2024-08-17T16:47:49.507 INFO:root:teuthology version: 1.2.0 2024-08-17T16:47:49.533 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T16:47:49.628 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7858331' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: main sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 17 sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 11/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 targets: smithi064.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGzVszy7jnI7Haxupq+X6ZEFX/FL1Z1Df53t9iUCqsWOimS+GvuN+LTl2StRgy8jmsGUMOCOzjwlgoqDdnTz9m0= smithi175.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBERO55NqU49bn88v2QmcwhoDUW7O4vueT6NVbUSu5NAqFkWDYRXuSSzpeQ0ZZ17deLzLbmGF60YpGUrDTZXNI6M= tasks: - pexec: all: - sudo dnf install nvmetcli nvme-cli -y - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 67db276914d29505c9707d04f3a3b8fb9d8aee4b timestamp: 2024-08-17_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-17T16:47:49.628 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa; will attempt to use it 2024-08-17T16:47:49.629 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks 2024-08-17T16:47:49.629 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-17T16:47:49.630 INFO:teuthology.task.internal:Checking packages... 2024-08-17T16:47:49.652 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453' 2024-08-17T16:47:49.652 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:47:49.652 INFO:teuthology.packaging:ref: None 2024-08-17T16:47:49.652 INFO:teuthology.packaging:tag: None 2024-08-17T16:47:49.653 INFO:teuthology.packaging:branch: main 2024-08-17T16:47:49.653 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:47:49.653 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-08-17T16:47:49.865 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-5842.g4887e20f 2024-08-17T16:47:49.867 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-17T16:47:49.873 INFO:teuthology.task.internal:no buildpackages task found 2024-08-17T16:47:49.873 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-17T16:47:49.897 INFO:teuthology.task.internal:Saving configuration 2024-08-17T16:47:49.917 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-17T16:47:49.931 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-17T16:47:49.964 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:42:48.887738', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGzVszy7jnI7Haxupq+X6ZEFX/FL1Z1Df53t9iUCqsWOimS+GvuN+LTl2StRgy8jmsGUMOCOzjwlgoqDdnTz9m0='} 2024-08-17T16:47:49.993 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi175.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:42:48.885693', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBERO55NqU49bn88v2QmcwhoDUW7O4vueT6NVbUSu5NAqFkWDYRXuSSzpeQ0ZZ17deLzLbmGF60YpGUrDTZXNI6M='} 2024-08-17T16:47:49.994 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-17T16:47:50.000 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-17T16:47:50.000 INFO:teuthology.task.internal:roles: ubuntu@smithi175.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-17T16:47:50.000 INFO:teuthology.run_tasks:Running task console_log... 2024-08-17T16:47:50.097 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8f11e47ac0>, signals=[15]) 2024-08-17T16:47:50.097 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-17T16:47:50.102 INFO:teuthology.task.internal:Opening connections... 2024-08-17T16:47:50.103 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2024-08-17T16:47:50.104 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:47:50.188 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi175.front.sepia.ceph.com 2024-08-17T16:47:50.189 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:47:50.288 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-17T16:47:50.294 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2024-08-17T16:47:50.312 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2024-08-17T16:47:50.313 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2024-08-17T16:47:50.370 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="CentOS Stream" 2024-08-17T16:47:50.370 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="9" 2024-08-17T16:47:50.370 INFO:teuthology.orchestra.run.smithi064.stdout:ID="centos" 2024-08-17T16:47:50.370 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:47:50.370 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="9" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:47:50.371 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:47:50.372 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2024-08-17T16:47:50.399 DEBUG:teuthology.orchestra.run.smithi175:> uname -m 2024-08-17T16:47:50.416 INFO:teuthology.orchestra.run.smithi175.stdout:x86_64 2024-08-17T16:47:50.416 DEBUG:teuthology.orchestra.run.smithi175:> cat /etc/os-release 2024-08-17T16:47:50.472 INFO:teuthology.orchestra.run.smithi175.stdout:NAME="CentOS Stream" 2024-08-17T16:47:50.472 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION="9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:ID="centos" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION_ID="9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:ANSI_COLOR="0;31" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:47:50.473 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:47:50.474 INFO:teuthology.lock.ops:Updating smithi175.front.sepia.ceph.com on lock server 2024-08-17T16:47:50.499 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-17T16:47:50.507 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-17T16:47:50.546 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-17T16:47:50.547 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:47:50.549 DEBUG:teuthology.orchestra.run.smithi175:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:47:50.565 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-17T16:47:50.571 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-17T16:47:50.571 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:47:50.606 DEBUG:teuthology.orchestra.run.smithi175:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:47:50.694 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-17T16:47:50.758 INFO:teuthology.run_tasks:Running task kernel... 2024-08-17T16:47:50.772 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-17T16:47:50.772 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-17T16:47:50.772 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:47:50.773 DEBUG:teuthology.orchestra.run.smithi175:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:47:50.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:47:50.790 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-08-17T16:47:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:47:50.791 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-08-17T16:47:50.847 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:47:50.847 INFO:teuthology.task.kernel:Running kernel on smithi175: 5.14.0-437.el9.x86_64 2024-08-17T16:47:50.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum install -y kernel 2024-08-17T16:47:50.849 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:47:50.849 INFO:teuthology.task.kernel:Running kernel on smithi064: 5.14.0-437.el9.x86_64 2024-08-17T16:47:50.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2024-08-17T16:47:53.258 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-08-17T16:47:54.450 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.2 MB 00:01 2024-08-17T16:47:55.434 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 19 MB/s | 20 MB 00:01 2024-08-17T16:47:56.950 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - AppStream 14 MB/s | 20 MB 00:01 2024-08-17T16:47:59.852 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.6 MB 00:00 2024-08-17T16:48:01.118 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - CRB 16 MB/s | 6.6 MB 00:00 2024-08-17T16:48:01.394 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 72 kB/s | 18 kB 00:00 2024-08-17T16:48:02.823 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 17 MB/s | 22 MB 00:01 2024-08-17T16:48:03.718 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 18 kB 00:01 2024-08-17T16:48:04.930 INFO:teuthology.orchestra.run.smithi175.stdout:Extra Packages for Enterprise Linux 21 MB/s | 22 MB 00:01 2024-08-17T16:48:08.503 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-08-17T16:48:10.302 INFO:teuthology.orchestra.run.smithi175.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-08-17T16:48:10.651 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:48:10.652 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:48:10.753 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:48:10.766 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:48:10.766 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-08-17T16:48:10.766 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:48:10.766 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:48:10.766 INFO:teuthology.orchestra.run.smithi064.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout:Install 4 Packages 2024-08-17T16:48:10.767 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:48:10.768 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 84 M 2024-08-17T16:48:10.768 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 126 M 2024-08-17T16:48:10.768 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:48:11.280 INFO:teuthology.orchestra.run.smithi064.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 6.4 MB/s | 1.5 MB 00:00 2024-08-17T16:48:11.580 INFO:teuthology.orchestra.run.smithi064.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 32 MB/s | 17 MB 00:00 2024-08-17T16:48:12.155 INFO:teuthology.orchestra.run.smithi064.stdout:(3/4): kernel-modules-core-5.14.0-480.el9.x86_6 34 MB/s | 30 MB 00:00 2024-08-17T16:48:12.283 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:48:12.284 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:48:12.391 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:48:12.397 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout:Install 4 Packages 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 84 M 2024-08-17T16:48:12.398 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 126 M 2024-08-17T16:48:12.399 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:48:12.703 INFO:teuthology.orchestra.run.smithi175.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 8.3 MB/s | 1.5 MB 00:00 2024-08-17T16:48:12.714 INFO:teuthology.orchestra.run.smithi064.stdout:(4/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 22 MB/s | 36 MB 00:01 2024-08-17T16:48:12.714 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:12.714 INFO:teuthology.orchestra.run.smithi064.stdout:Total 43 MB/s | 84 MB 00:01 2024-08-17T16:48:13.037 INFO:teuthology.orchestra.run.smithi175.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 33 MB/s | 17 MB 00:00 2024-08-17T16:48:13.421 INFO:teuthology.orchestra.run.smithi175.stdout:(3/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 40 MB/s | 36 MB 00:00 2024-08-17T16:48:13.477 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:48:13.713 INFO:teuthology.orchestra.run.smithi175.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 30 MB/s | 30 MB 00:01 2024-08-17T16:48:13.713 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:13.713 INFO:teuthology.orchestra.run.smithi175.stdout:Total 64 MB/s | 84 MB 00:01 2024-08-17T16:48:13.904 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:48:13.904 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:48:14.452 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:48:14.788 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:48:14.788 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:48:14.924 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:48:14.925 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:48:15.788 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:48:15.789 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:48:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:48:18.219 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:48:18.239 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:48:18.339 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:48:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:48:18.979 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:48:21.133 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:48:21.289 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:48:21.619 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:48:21.770 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:48:26.502 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:48:26.597 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:48:26.671 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:48:26.695 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:48:31.535 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:48:31.897 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:01.351 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:01.391 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:01.782 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:01.782 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:49:01.782 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:49:01.782 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:49:02.581 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:02.581 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:49:02.581 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:49:02.581 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:02.581 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:49:02.582 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:49:02.582 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:49:02.582 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:49:02.582 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:49:02.934 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:49:03.017 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:03.017 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:49:03.018 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:49:03.018 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:49:03.018 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:49:03.018 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-08-17T16:49:03.018 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi175.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:49:03.018 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:49:03.075 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:03.076 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:49:03.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo rpm -qi grub2-tools 2024-08-17T16:49:03.116 INFO:teuthology.orchestra.run.smithi175.stdout:Name : grub2-tools 2024-08-17T16:49:03.116 INFO:teuthology.orchestra.run.smithi175.stdout:Epoch : 1 2024-08-17T16:49:03.116 INFO:teuthology.orchestra.run.smithi175.stdout:Version : 2.06 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Release : 68.el9 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Architecture: x86_64 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Group : Unspecified 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Size : 8264254 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:License : GPLv3+ 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Packager : builder@centos.org 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:Vendor : CentOS 2024-08-17T16:49:03.117 INFO:teuthology.orchestra.run.smithi175.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:Summary : Support tools for GRUB. 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:Description : 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:hardware devices. 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:49:03.118 INFO:teuthology.orchestra.run.smithi175.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:49:03.119 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:49:03.119 INFO:teuthology.task.kernel:Updating grub on smithi175 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:49:03.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:49:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:03.587 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:03.587 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:49:03.587 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:49:03.587 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:49:03.826 INFO:teuthology.orchestra.run.smithi175.stderr:Generating grub configuration file ... 2024-08-17T16:49:04.437 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:49:04.437 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:49:04.437 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:49:04.438 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:49:04.830 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:49:04.917 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.917 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:49:04.917 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:49:04.917 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:49:04.917 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:49:04.918 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-08-17T16:49:04.918 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi064.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:49:04.918 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:49:04.977 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:49:04.977 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:49:04.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo rpm -qi grub2-tools 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Name : grub2-tools 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Epoch : 1 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Version : 2.06 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Release : 68.el9 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Architecture: x86_64 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Group : Unspecified 2024-08-17T16:49:05.008 INFO:teuthology.orchestra.run.smithi064.stdout:Size : 8264254 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:License : GPLv3+ 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Packager : builder@centos.org 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Vendor : CentOS 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Summary : Support tools for GRUB. 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:Description : 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout:hardware devices. 2024-08-17T16:49:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:49:05.010 INFO:teuthology.orchestra.run.smithi064.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:49:05.011 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:49:05.011 INFO:teuthology.task.kernel:Updating grub on smithi064 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:49:05.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:49:05.317 INFO:teuthology.orchestra.run.smithi175.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:49:05.349 INFO:teuthology.orchestra.run.smithi175.stderr:done 2024-08-17T16:49:05.351 DEBUG:teuthology.orchestra.run.smithi175:> mktemp 2024-08-17T16:49:05.366 INFO:teuthology.orchestra.run.smithi175.stdout:/tmp/tmp.tRGK4QsJjL 2024-08-17T16:49:05.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.tRGK4QsJjL 2024-08-17T16:49:05.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 0666 /tmp/tmp.tRGK4QsJjL 2024-08-17T16:49:05.578 DEBUG:teuthology.orchestra.remote:smithi175:/tmp/tmp.tRGK4QsJjL is 6KB 2024-08-17T16:49:05.627 DEBUG:teuthology.orchestra.run.smithi175:> rm -fr /tmp/tmp.tRGK4QsJjL 2024-08-17T16:49:05.629 INFO:teuthology.orchestra.run.smithi064.stderr:Generating grub configuration file ... 2024-08-17T16:49:05.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:49:05.703 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:49:05.703 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:49:05.703 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:49:05.703 INFO:teuthology.orchestra.run.smithi175.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:49:05.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:49:05.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo shutdown -r now 2024-08-17T16:49:07.065 INFO:teuthology.orchestra.run.smithi064.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:49:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:done 2024-08-17T16:49:07.094 DEBUG:teuthology.orchestra.run.smithi064:> mktemp 2024-08-17T16:49:07.110 INFO:teuthology.orchestra.run.smithi064.stdout:/tmp/tmp.uO5faO4DHX 2024-08-17T16:49:07.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.uO5faO4DHX 2024-08-17T16:49:07.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0666 /tmp/tmp.uO5faO4DHX 2024-08-17T16:49:07.312 DEBUG:teuthology.orchestra.remote:smithi064:/tmp/tmp.uO5faO4DHX is 6KB 2024-08-17T16:49:07.359 DEBUG:teuthology.orchestra.run.smithi064:> rm -fr /tmp/tmp.uO5faO4DHX 2024-08-17T16:49:07.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:49:07.436 INFO:teuthology.orchestra.run.smithi064.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:49:07.437 INFO:teuthology.orchestra.run.smithi064.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:49:07.437 INFO:teuthology.orchestra.run.smithi064.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:49:07.437 INFO:teuthology.orchestra.run.smithi064.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:49:07.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:49:07.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo shutdown -r now 2024-08-17T16:49:35.886 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-08-17T16:49:35.890 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:49:35.891 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:49:37.576 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-08-17T16:49:37.577 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:49:37.577 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:49:54.403 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.175 2024-08-17T16:49:55.937 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-08-17T16:50:03.410 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:50:03.411 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:04.939 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:50:04.940 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:06.498 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.175 2024-08-17T16:50:08.002 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-08-17T16:50:18.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:50:18.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:20.005 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:50:20.006 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:36.895 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.175 2024-08-17T16:50:38.430 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.64 2024-08-17T16:50:51.906 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:50:51.907 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:52.209 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-08-17T16:50:52.573 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:50:52.573 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:50:52.573 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-08-17T16:50:52.589 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:50:52.589 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:50:52.589 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:50:52.590 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:50:53.432 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:50:53.432 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:50:53.590 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-17T16:50:53.591 DEBUG:teuthology.orchestra.run.smithi175:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:50:53.698 INFO:teuthology.orchestra.run.smithi175.stdout:ttyS1 2024-08-17T16:50:53.740 DEBUG:teuthology.parallel:result is None 2024-08-17T16:50:53.901 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-08-17T16:50:54.326 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:50:54.327 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:50:54.327 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2024-08-17T16:50:54.342 INFO:teuthology.orchestra.run.smithi064.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:50:54.342 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:50:54.343 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:50:54.343 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:50:55.343 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-17T16:50:55.344 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:50:55.477 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2024-08-17T16:50:55.517 DEBUG:teuthology.parallel:result is None 2024-08-17T16:50:55.517 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-17T16:50:55.524 INFO:teuthology.task.internal:Creating test directory... 2024-08-17T16:50:55.525 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:50:55.527 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:50:55.565 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-17T16:50:55.576 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-17T16:50:55.619 INFO:teuthology.task.internal:Creating archive directory... 2024-08-17T16:50:55.619 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:50:55.622 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:50:55.668 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-17T16:50:55.679 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-17T16:50:55.680 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:50:55.709 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:50:55.746 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:50:55.757 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:50:55.782 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:50:55.805 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:50:55.808 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-17T16:50:55.819 INFO:teuthology.task.internal:Configuring sudo... 2024-08-17T16:50:55.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:50:55.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:50:55.880 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-17T16:50:55.892 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-17T16:50:55.893 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:50:55.895 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:50:55.938 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:50:56.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:50:56.147 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:50:56.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:50:56.260 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:50:56.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:50:56.330 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:50:56.381 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:50:56.423 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:50:56.513 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:50:56.548 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:50:56.548 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:50:56.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2024-08-17T16:50:56.619 DEBUG:teuthology.orchestra.run.smithi175:> sudo service rsyslog restart 2024-08-17T16:50:56.665 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:50:56.696 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:50:57.183 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-17T16:50:57.192 INFO:teuthology.task.internal:Starting timer... 2024-08-17T16:50:57.192 INFO:teuthology.run_tasks:Running task pcp... 2024-08-17T16:50:57.245 INFO:teuthology.run_tasks:Running task selinux... 2024-08-17T16:50:57.296 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-08-17T16:50:57.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2024-08-17T16:50:57.340 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2024-08-17T16:50:57.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo service auditd rotate 2024-08-17T16:50:57.404 INFO:teuthology.orchestra.run.smithi175.stdout:Rotating logs: 2024-08-17T16:50:57.406 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-17T16:50:57.407 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2024-08-17T16:50:57.428 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2024-08-17T16:50:57.428 DEBUG:teuthology.orchestra.run.smithi175:> /usr/sbin/getenforce 2024-08-17T16:50:57.452 INFO:teuthology.orchestra.run.smithi175.stdout:Permissive 2024-08-17T16:50:57.453 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi064.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi175.front.sepia.ceph.com': 'permissive'} 2024-08-17T16:50:57.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-17T16:50:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:50:57.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-17T16:50:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:50:57.526 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-17T16:50:57.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:50:57.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:50:57.593 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-17T16:50:57.603 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-17T16:50:57.616 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-17T16:50:57.616 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi064.front.sepia.ceph.com,smithi175.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-17T16:55:38.296 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi064.front.sepia.ceph.com'), Remote(name='ubuntu@smithi175.front.sepia.ceph.com')] 2024-08-17T16:55:38.297 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:55:38.298 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:55:38.386 DEBUG:teuthology.orchestra.run.smithi064:> true 2024-08-17T16:55:38.471 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2024-08-17T16:55:38.471 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:55:38.472 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:55:38.551 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-08-17T16:55:38.637 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-08-17T16:55:38.637 INFO:teuthology.run_tasks:Running task clock... 2024-08-17T16:55:38.648 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-17T16:55:38.648 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:55:38.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:55:38.651 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:55:38.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:55:38.680 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:55:38.695 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:55:38.706 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:55:38.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2024-08-17T16:55:38.721 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:55:38.733 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2024-08-17T16:55:38.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: ntpd: command not found 2024-08-17T16:55:38.747 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:55:38.755 INFO:teuthology.orchestra.run.smithi175.stdout:506 Cannot talk to daemon 2024-08-17T16:55:38.762 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:55:38.770 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:55:38.783 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:55:38.816 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:55:38.819 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:55:38.819 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-08-17T16:55:38.819 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.819 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.819 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.820 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.836 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:55:38.839 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:55:38.839 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-08-17T16:55:38.840 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.840 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.840 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.840 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:55:38.841 INFO:teuthology.run_tasks:Running task pexec... 2024-08-17T16:55:38.851 INFO:teuthology.task.pexec:Executing custom commands... 2024-08-17T16:55:38.851 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi064.front.sepia.ceph.com 2024-08-17T16:55:38.851 DEBUG:teuthology.orchestra.run.smithi064:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:55:38.852 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi175.front.sepia.ceph.com 2024-08-17T16:55:38.852 DEBUG:teuthology.orchestra.run.smithi175:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:55:39.447 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:17 ago on Sat 17 Aug 2024 04:54:22 PM UTC. 2024-08-17T16:55:39.450 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:11 ago on Sat 17 Aug 2024 04:54:28 PM UTC. 2024-08-17T16:55:39.530 INFO:teuthology.orchestra.run.smithi175.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:55:39.535 INFO:teuthology.orchestra.run.smithi064.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:55:39.571 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:55:39.580 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:55:39.582 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:55:39.582 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-08-17T16:55:39.582 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:55:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:Install 4 Packages 2024-08-17T16:55:39.584 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 2 Packages 2024-08-17T16:55:39.584 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:55:39.584 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 2.0 M 2024-08-17T16:55:39.584 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:55:39.586 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:55:39.586 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-08-17T16:55:39.586 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:55:39.586 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:55:39.586 INFO:teuthology.orchestra.run.smithi175.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:Upgrading: 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:Install 4 Packages 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout:Upgrade 2 Packages 2024-08-17T16:55:39.587 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:55:39.588 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 2.0 M 2024-08-17T16:55:39.588 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:55:39.971 INFO:teuthology.orchestra.run.smithi175.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 389 kB/s | 44 kB 00:00 2024-08-17T16:55:39.996 INFO:teuthology.orchestra.run.smithi175.stdout:(2/6): python3-kmod-0.9-32.el9.x86_64.rpm 603 kB/s | 84 kB 00:00 2024-08-17T16:55:40.004 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 125 kB/s | 44 kB 00:00 2024-08-17T16:55:40.022 INFO:teuthology.orchestra.run.smithi175.stdout:(3/6): python3-configshell-1.1.30-1.el9.noarch. 435 kB/s | 72 kB 00:00 2024-08-17T16:55:40.029 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): python3-configshell-1.1.30-1.el9.noarch. 189 kB/s | 72 kB 00:00 2024-08-17T16:55:40.055 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): python3-kmod-0.9-32.el9.x86_64.rpm 208 kB/s | 84 kB 00:00 2024-08-17T16:55:40.105 INFO:teuthology.orchestra.run.smithi175.stdout:(4/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 6.1 MB/s | 837 kB 00:00 2024-08-17T16:55:40.131 INFO:teuthology.orchestra.run.smithi175.stdout:(5/6): libnvme-1.9-2.el9.x86_64.rpm 793 kB/s | 106 kB 00:00 2024-08-17T16:55:40.189 INFO:teuthology.orchestra.run.smithi175.stdout:(6/6): nvme-cli-2.9.1-3.el9.x86_64.rpm 5.3 MB/s | 903 kB 00:00 2024-08-17T16:55:40.190 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:55:40.190 INFO:teuthology.orchestra.run.smithi175.stdout:Total 3.3 MB/s | 2.0 MB 00:00 2024-08-17T16:55:40.222 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): libnvme-1.9-2.el9.x86_64.rpm 552 kB/s | 106 kB 00:00 2024-08-17T16:55:40.262 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:55:40.278 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:55:40.278 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:55:40.413 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): nvme-cli-2.9.1-3.el9.x86_64.rpm 2.5 MB/s | 903 kB 00:00 2024-08-17T16:55:40.439 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 1.9 MB/s | 837 kB 00:00 2024-08-17T16:55:40.439 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:55:40.439 INFO:teuthology.orchestra.run.smithi064.stdout:Total 2.3 MB/s | 2.0 MB 00:00 2024-08-17T16:55:40.484 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:55:40.484 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:55:40.518 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:55:40.535 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:55:40.535 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:55:40.736 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:55:40.736 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:55:40.833 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:55:41.049 INFO:teuthology.orchestra.run.smithi175.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/8 2024-08-17T16:55:41.101 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/8 2024-08-17T16:55:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:55:41.145 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/8 2024-08-17T16:55:41.178 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/8 2024-08-17T16:55:41.203 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/8 2024-08-17T16:55:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/8 2024-08-17T16:55:41.420 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/8 2024-08-17T16:55:41.445 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/8 2024-08-17T16:55:41.479 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/8 2024-08-17T16:55:41.496 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/8 2024-08-17T16:55:41.511 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/8 2024-08-17T16:55:41.564 INFO:teuthology.orchestra.run.smithi175.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/8 2024-08-17T16:55:41.771 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/8 2024-08-17T16:55:41.839 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/8 2024-08-17T16:55:41.842 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/8 2024-08-17T16:55:41.875 INFO:teuthology.orchestra.run.smithi175.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 7/8 2024-08-17T16:55:41.969 INFO:teuthology.orchestra.run.smithi175.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 8/8 2024-08-17T16:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/8 2024-08-17T16:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 7/8 2024-08-17T16:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 8/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 8/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/8 2024-08-17T16:55:42.775 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libnvme-1.9-2.el9.x86_64 5/8 2024-08-17T16:55:42.776 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libnvme-1.6-1.el9.x86_64 6/8 2024-08-17T16:55:42.776 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 7/8 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 8/8 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout:Upgraded: 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:55:43.100 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:55:43.101 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 8/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libnvme-1.9-2.el9.x86_64 5/8 2024-08-17T16:55:43.150 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libnvme-1.6-1.el9.x86_64 6/8 2024-08-17T16:55:43.151 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 7/8 2024-08-17T16:55:43.480 DEBUG:teuthology.parallel:result is None 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 8/8 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:55:43.513 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:55:43.856 DEBUG:teuthology.parallel:result is None 2024-08-17T16:55:43.857 INFO:teuthology.run_tasks:Running task install... 2024-08-17T16:55:43.865 DEBUG:teuthology.task.install:project ceph 2024-08-17T16:55:43.865 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}} 2024-08-17T16:55:43.865 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:55:43.866 INFO:teuthology.task.install:Using flavor: default 2024-08-17T16:55:43.871 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-17T16:55:43.871 INFO:teuthology.task.install:extra packages: [] 2024-08-17T16:55:43.871 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:55:43.872 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:55:43.872 INFO:teuthology.packaging:ref: None 2024-08-17T16:55:43.872 INFO:teuthology.packaging:tag: None 2024-08-17T16:55:43.872 INFO:teuthology.packaging:branch: reef 2024-08-17T16:55:43.872 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:43.872 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:55:43.874 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:55:43.875 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:55:43.875 INFO:teuthology.packaging:ref: None 2024-08-17T16:55:43.875 INFO:teuthology.packaging:tag: None 2024-08-17T16:55:43.875 INFO:teuthology.packaging:branch: reef 2024-08-17T16:55:43.875 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:43.875 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:55:44.117 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/ 2024-08-17T16:55:44.119 INFO:teuthology.task.install.rpm:Package version is 18.2.4-719.gced649f8 2024-08-17T16:55:44.157 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/ 2024-08-17T16:55:44.157 INFO:teuthology.task.install.rpm:Package version is 18.2.4-719.gced649f8 2024-08-17T16:55:44.263 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:55:44.263 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:55:44.263 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:55:44.278 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:55:44.279 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:55:44.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:55:44.291 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:55:44.292 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:55:44.292 INFO:teuthology.packaging:ref: None 2024-08-17T16:55:44.292 INFO:teuthology.packaging:tag: None 2024-08-17T16:55:44.292 INFO:teuthology.packaging:branch: reef 2024-08-17T16:55:44.292 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:44.292 DEBUG:teuthology.orchestra.run.smithi175:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:55:44.308 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:55:44.309 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:55:44.309 INFO:teuthology.packaging:ref: None 2024-08-17T16:55:44.309 INFO:teuthology.packaging:tag: None 2024-08-17T16:55:44.309 INFO:teuthology.packaging:branch: reef 2024-08-17T16:55:44.309 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:44.309 DEBUG:teuthology.orchestra.run.smithi064:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:55:44.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:55:44.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:55:44.433 DEBUG:teuthology.orchestra.run.smithi175:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:55:44.440 DEBUG:teuthology.orchestra.run.smithi064:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:55:44.498 INFO:teuthology.orchestra.run.smithi175.stdout:check_obsoletes = 1 2024-08-17T16:55:44.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum clean all 2024-08-17T16:55:44.505 INFO:teuthology.orchestra.run.smithi064.stdout:check_obsoletes = 1 2024-08-17T16:55:44.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum clean all 2024-08-17T16:55:44.727 INFO:teuthology.orchestra.run.smithi175.stdout:45 files removed 2024-08-17T16:55:44.745 INFO:teuthology.orchestra.run.smithi064.stdout:45 files removed 2024-08-17T16:55:44.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-radosgw 2024-08-17T16:55:44.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-radosgw 2024-08-17T16:55:45.348 INFO:teuthology.orchestra.run.smithi175.stdout:ceph packages for x86_64 245 kB/s | 77 kB 00:00 2024-08-17T16:55:45.372 INFO:teuthology.orchestra.run.smithi064.stdout:ceph packages for x86_64 250 kB/s | 77 kB 00:00 2024-08-17T16:55:45.690 INFO:teuthology.orchestra.run.smithi175.stdout:ceph noarch packages 39 kB/s | 11 kB 00:00 2024-08-17T16:55:45.714 INFO:teuthology.orchestra.run.smithi064.stdout:ceph noarch packages 40 kB/s | 11 kB 00:00 2024-08-17T16:55:46.032 INFO:teuthology.orchestra.run.smithi175.stdout:ceph source packages 6.3 kB/s | 1.9 kB 00:00 2024-08-17T16:55:46.048 INFO:teuthology.orchestra.run.smithi064.stdout:ceph source packages 6.5 kB/s | 1.9 kB 00:00 2024-08-17T16:55:46.516 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - BaseOS 19 MB/s | 8.2 MB 00:00 2024-08-17T16:55:47.799 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.2 MB 00:01 2024-08-17T16:55:48.225 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - AppStream 34 MB/s | 20 MB 00:00 2024-08-17T16:55:49.791 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - AppStream 24 MB/s | 20 MB 00:00 2024-08-17T16:55:53.243 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - CRB 5.4 MB/s | 6.6 MB 00:01 2024-08-17T16:55:54.167 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.6 MB 00:00 2024-08-17T16:55:54.735 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 18 kB 00:00 2024-08-17T16:55:55.643 INFO:teuthology.orchestra.run.smithi064.stdout:CentOS Stream 9 - Extras packages 87 kB/s | 18 kB 00:00 2024-08-17T16:55:55.673 INFO:teuthology.orchestra.run.smithi175.stdout:Extra Packages for Enterprise Linux 28 MB/s | 22 MB 00:00 2024-08-17T16:55:56.814 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 22 MB/s | 22 MB 00:01 2024-08-17T16:56:01.262 INFO:teuthology.orchestra.run.smithi175.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-08-17T16:56:02.462 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-17T16:56:03.578 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-radosgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 7.7 M 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout:Upgrading: 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: librados2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.3 M 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: librbd1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:56:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-base x86_64 2:18.2.4-719.gced649f8.el9 ceph 5.1 M 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 18 M 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-selinux x86_64 2:18.2.4-719.gced649f8.el9 ceph 25 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: libcephfs2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 712 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: libradosstriper1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 477 k 2024-08-17T16:56:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: librgw2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.5 M 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ceph-argparse x86_64 2:18.2.4-719.gced649f8.el9 ceph 45 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 129 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cephfs x86_64 2:18.2.4-719.gced649f8.el9 ceph 188 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rados x86_64 2:18.2.4-719.gced649f8.el9 ceph 372 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 347 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 116 k 2024-08-17T16:56:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout:Install 28 Packages 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout:Upgrade 2 Packages 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:03.583 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 53 M 2024-08-17T16:56:03.584 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:04.271 INFO:teuthology.orchestra.run.smithi175.stdout:(1/30): ceph-base-18.2.4-719.gced649f8.el9.x86_ 9.8 MB/s | 5.1 MB 00:00 2024-08-17T16:56:04.321 INFO:teuthology.orchestra.run.smithi175.stdout:(2/30): ceph-selinux-18.2.4-719.gced649f8.el9.x 506 kB/s | 25 kB 00:00 2024-08-17T16:56:04.413 INFO:teuthology.orchestra.run.smithi175.stdout:(3/30): ceph-radosgw-18.2.4-719.gced649f8.el9.x 12 MB/s | 7.7 MB 00:00 2024-08-17T16:56:04.446 INFO:teuthology.orchestra.run.smithi175.stdout:(4/30): libcephfs2-18.2.4-719.gced649f8.el9.x86 5.6 MB/s | 712 kB 00:00 2024-08-17T16:56:04.480 INFO:teuthology.orchestra.run.smithi175.stdout:(5/30): libradosstriper1-18.2.4-719.gced649f8.e 7.0 MB/s | 477 kB 00:00 2024-08-17T16:56:04.555 INFO:teuthology.orchestra.run.smithi175.stdout:(6/30): python3-ceph-argparse-18.2.4-719.gced64 600 kB/s | 45 kB 00:00 2024-08-17T16:56:04.656 INFO:teuthology.orchestra.run.smithi175.stdout:(7/30): librgw2-18.2.4-719.gced649f8.el9.x86_64 21 MB/s | 4.5 MB 00:00 2024-08-17T16:56:04.681 INFO:teuthology.orchestra.run.smithi175.stdout:(8/30): python3-ceph-common-18.2.4-719.gced649f 1.0 MB/s | 129 kB 00:00 2024-08-17T16:56:04.706 INFO:teuthology.orchestra.run.smithi175.stdout:(9/30): python3-cephfs-18.2.4-719.gced649f8.el9 3.7 MB/s | 188 kB 00:00 2024-08-17T16:56:04.740 INFO:teuthology.orchestra.run.smithi175.stdout:(10/30): python3-rados-18.2.4-719.gced649f8.el9 6.2 MB/s | 372 kB 00:00 2024-08-17T16:56:04.765 INFO:teuthology.orchestra.run.smithi175.stdout:(11/30): python3-rbd-18.2.4-719.gced649f8.el9.x 5.8 MB/s | 347 kB 00:00 2024-08-17T16:56:04.790 INFO:teuthology.orchestra.run.smithi175.stdout:(12/30): python3-rgw-18.2.4-719.gced649f8.el9.x 2.3 MB/s | 116 kB 00:00 2024-08-17T16:56:04.873 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:04.875 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:56:04.875 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:04.875 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 7.7 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: librados2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.3 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base x86_64 2:18.2.4-719.gced649f8.el9 ceph 5.1 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 18 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux x86_64 2:18.2.4-719.gced649f8.el9 ceph 25 k 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-17T16:56:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 712 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 477 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.5 M 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse x86_64 2:18.2.4-719.gced649f8.el9 ceph 45 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 129 k 2024-08-17T16:56:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs x86_64 2:18.2.4-719.gced649f8.el9 ceph 188 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados x86_64 2:18.2.4-719.gced649f8.el9 ceph 372 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 347 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 116 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout:Install 28 Packages 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 2 Packages 2024-08-17T16:56:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:04.879 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 53 M 2024-08-17T16:56:04.879 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:04.966 INFO:teuthology.orchestra.run.smithi175.stdout:(13/30): ceph-common-18.2.4-719.gced649f8.el9.x 15 MB/s | 18 MB 00:01 2024-08-17T16:56:05.016 INFO:teuthology.orchestra.run.smithi175.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 427 kB/s | 107 kB 00:00 2024-08-17T16:56:05.125 INFO:teuthology.orchestra.run.smithi175.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 286 kB/s | 45 kB 00:00 2024-08-17T16:56:05.150 INFO:teuthology.orchestra.run.smithi175.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 446 kB/s | 160 kB 00:00 2024-08-17T16:56:05.175 INFO:teuthology.orchestra.run.smithi175.stdout:(17/30): librdkafka-1.6.1-102.el9.x86_64.rpm 4.1 MB/s | 662 kB 00:00 2024-08-17T16:56:05.225 INFO:teuthology.orchestra.run.smithi175.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 561 kB/s | 42 kB 00:00 2024-08-17T16:56:05.251 INFO:teuthology.orchestra.run.smithi175.stdout:(19/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.3 MB/s | 292 kB 00:00 2024-08-17T16:56:05.326 INFO:teuthology.orchestra.run.smithi175.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 736 kB/s | 111 kB 00:00 2024-08-17T16:56:05.351 INFO:teuthology.orchestra.run.smithi175.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-08-17T16:56:05.385 INFO:teuthology.orchestra.run.smithi175.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.4 MB/s | 49 kB 00:00 2024-08-17T16:56:05.410 INFO:teuthology.orchestra.run.smithi175.stdout:(23/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.7 MB/s | 309 kB 00:00 2024-08-17T16:56:05.435 INFO:teuthology.orchestra.run.smithi175.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-08-17T16:56:05.469 INFO:teuthology.orchestra.run.smithi175.stdout:(25/30): re2-20211101-3.el9.x86_64.rpm 5.6 MB/s | 192 kB 00:00 2024-08-17T16:56:05.503 INFO:teuthology.orchestra.run.smithi175.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.9 MB/s | 839 kB 00:00 2024-08-17T16:56:05.578 INFO:teuthology.orchestra.run.smithi175.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 15 MB/s | 1.6 MB 00:00 2024-08-17T16:56:05.703 INFO:teuthology.orchestra.run.smithi175.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2024-08-17T16:56:05.753 INFO:teuthology.orchestra.run.smithi175.stdout:(29/30): librados2-18.2.4-719.gced649f8.el9.x86 13 MB/s | 3.3 MB 00:00 2024-08-17T16:56:05.812 INFO:teuthology.orchestra.run.smithi175.stdout:(30/30): librbd1-18.2.4-719.gced649f8.el9.x86_6 13 MB/s | 3.0 MB 00:00 2024-08-17T16:56:05.816 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:05.817 INFO:teuthology.orchestra.run.smithi175.stdout:Total 24 MB/s | 53 MB 00:02 2024-08-17T16:56:05.886 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:05.920 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:05.921 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:06.037 INFO:teuthology.orchestra.run.smithi064.stdout:(1/30): ceph-base-18.2.4-719.gced649f8.el9.x86_ 7.7 MB/s | 5.1 MB 00:00 2024-08-17T16:56:06.088 INFO:teuthology.orchestra.run.smithi064.stdout:(2/30): ceph-selinux-18.2.4-719.gced649f8.el9.x 504 kB/s | 25 kB 00:00 2024-08-17T16:56:06.172 INFO:teuthology.orchestra.run.smithi064.stdout:(3/30): libcephfs2-18.2.4-719.gced649f8.el9.x86 8.4 MB/s | 712 kB 00:00 2024-08-17T16:56:06.230 INFO:teuthology.orchestra.run.smithi064.stdout:(4/30): libradosstriper1-18.2.4-719.gced649f8.e 8.0 MB/s | 477 kB 00:00 2024-08-17T16:56:06.313 INFO:teuthology.orchestra.run.smithi064.stdout:(5/30): ceph-radosgw-18.2.4-719.gced649f8.el9.x 8.3 MB/s | 7.7 MB 00:00 2024-08-17T16:56:06.359 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:06.359 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:06.363 INFO:teuthology.orchestra.run.smithi064.stdout:(6/30): python3-ceph-argparse-18.2.4-719.gced64 903 kB/s | 45 kB 00:00 2024-08-17T16:56:06.414 INFO:teuthology.orchestra.run.smithi064.stdout:(7/30): python3-ceph-common-18.2.4-719.gced649f 2.5 MB/s | 129 kB 00:00 2024-08-17T16:56:06.473 INFO:teuthology.orchestra.run.smithi064.stdout:(8/30): python3-cephfs-18.2.4-719.gced649f8.el9 3.1 MB/s | 188 kB 00:00 2024-08-17T16:56:06.531 INFO:teuthology.orchestra.run.smithi064.stdout:(9/30): librgw2-18.2.4-719.gced649f8.el9.x86_64 15 MB/s | 4.5 MB 00:00 2024-08-17T16:56:06.565 INFO:teuthology.orchestra.run.smithi064.stdout:(10/30): python3-rados-18.2.4-719.gced649f8.el9 4.0 MB/s | 372 kB 00:00 2024-08-17T16:56:06.590 INFO:teuthology.orchestra.run.smithi064.stdout:(11/30): python3-rbd-18.2.4-719.gced649f8.el9.x 5.8 MB/s | 347 kB 00:00 2024-08-17T16:56:06.615 INFO:teuthology.orchestra.run.smithi064.stdout:(12/30): python3-rgw-18.2.4-719.gced649f8.el9.x 2.3 MB/s | 116 kB 00:00 2024-08-17T16:56:06.924 INFO:teuthology.orchestra.run.smithi064.stdout:(13/30): ceph-common-18.2.4-719.gced649f8.el9.x 12 MB/s | 18 MB 00:01 2024-08-17T16:56:07.474 INFO:teuthology.orchestra.run.smithi064.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 121 kB/s | 107 kB 00:00 2024-08-17T16:56:07.575 INFO:teuthology.orchestra.run.smithi064.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 72 kB/s | 45 kB 00:00 2024-08-17T16:56:07.608 INFO:teuthology.orchestra.run.smithi064.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 161 kB/s | 160 kB 00:00 2024-08-17T16:56:07.710 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-17T16:56:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-17T16:56:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-17T16:56:08.208 INFO:teuthology.orchestra.run.smithi064.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 70 kB/s | 42 kB 00:00 2024-08-17T16:56:08.211 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-17T16:56:08.292 INFO:teuthology.orchestra.run.smithi064.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 1.3 MB/s | 111 kB 00:00 2024-08-17T16:56:08.316 INFO:teuthology.orchestra.run.smithi175.stdout: Upgrading : librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:08.326 INFO:teuthology.orchestra.run.smithi064.stdout:(19/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 9.1 MB/s | 309 kB 00:00 2024-08-17T16:56:08.411 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:08.494 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:56:08.518 INFO:teuthology.orchestra.run.smithi064.stdout:(20/30): libarrow-9.0.0-11.el9.x86_64.rpm 23 MB/s | 4.4 MB 00:00 2024-08-17T16:56:08.543 INFO:teuthology.orchestra.run.smithi064.stdout:(21/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 302 kB/s | 292 kB 00:00 2024-08-17T16:56:08.568 INFO:teuthology.orchestra.run.smithi064.stdout:(22/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 518 kB/s | 26 kB 00:00 2024-08-17T16:56:08.619 INFO:teuthology.orchestra.run.smithi064.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.9 MB/s | 67 kB 00:00 2024-08-17T16:56:08.652 INFO:teuthology.orchestra.run.smithi064.stdout:(24/30): liboath-2.6.7-2.el9.x86_64.rpm 446 kB/s | 49 kB 00:00 2024-08-17T16:56:08.694 INFO:teuthology.orchestra.run.smithi064.stdout:(25/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 11 MB/s | 839 kB 00:00 2024-08-17T16:56:08.701 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:08.720 INFO:teuthology.orchestra.run.smithi064.stdout:(26/30): re2-20211101-3.el9.x86_64.rpm 2.8 MB/s | 192 kB 00:00 2024-08-17T16:56:08.763 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:08.786 INFO:teuthology.orchestra.run.smithi064.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 17 MB/s | 1.6 MB 00:00 2024-08-17T16:56:08.791 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-17T16:56:08.837 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 9/32 2024-08-17T16:56:08.902 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:56:08.927 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:08.978 INFO:teuthology.orchestra.run.smithi064.stdout:(28/30): librdkafka-1.6.1-102.el9.x86_64.rpm 441 kB/s | 662 kB 00:01 2024-08-17T16:56:08.993 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:09.037 INFO:teuthology.orchestra.run.smithi064.stdout:(29/30): librados2-18.2.4-719.gced649f8.el9.x86 10 MB/s | 3.3 MB 00:00 2024-08-17T16:56:09.095 INFO:teuthology.orchestra.run.smithi064.stdout:(30/30): librbd1-18.2.4-719.gced649f8.el9.x86_6 9.8 MB/s | 3.0 MB 00:00 2024-08-17T16:56:09.098 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-08-17T16:56:09.101 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:09.101 INFO:teuthology.orchestra.run.smithi064.stdout:Total 13 MB/s | 53 MB 00:04 2024-08-17T16:56:09.126 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-17T16:56:09.167 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-17T16:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:56:09.199 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-17T16:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:56:09.226 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-17T16:56:09.263 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-17T16:56:09.295 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-17T16:56:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-17T16:56:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: Upgrading : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:56:09.685 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:56:09.685 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:56:09.727 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:56:09.754 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 21/32 2024-08-17T16:56:09.790 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-17T16:56:10.178 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-17T16:56:10.219 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:56:10.292 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:56:10.367 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 25/32 2024-08-17T16:56:10.396 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 26/32 2024-08-17T16:56:11.017 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:56:11.097 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-17T16:56:11.126 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-17T16:56:11.182 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-17T16:56:11.532 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-17T16:56:11.625 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:11.705 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:11.796 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:56:11.826 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:11.889 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:11.917 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-17T16:56:11.956 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 9/32 2024-08-17T16:56:12.021 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:56:12.045 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:12.111 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:12.223 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-08-17T16:56:12.260 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-17T16:56:12.301 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-17T16:56:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-17T16:56:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-17T16:56:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-17T16:56:12.438 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-17T16:56:12.471 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:12.510 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-17T16:56:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:56:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:56:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 21/32 2024-08-17T16:56:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-17T16:56:13.038 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:13.080 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:56:13.196 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:56:13.196 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:56:13.196 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:56:13.197 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:13.222 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:13.372 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-17T16:56:13.394 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:56:13.467 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:56:13.537 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 25/32 2024-08-17T16:56:13.577 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 26/32 2024-08-17T16:56:15.769 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:15.805 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:16.368 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:56:16.408 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:56:16.516 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:56:16.516 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:56:16.516 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:56:16.516 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:22.853 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:22.853 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /sys 2024-08-17T16:56:22.853 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /proc 2024-08-17T16:56:22.853 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /mnt 2024-08-17T16:56:22.854 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /var/tmp 2024-08-17T16:56:22.854 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /home 2024-08-17T16:56:22.854 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /root 2024-08-17T16:56:22.854 INFO:teuthology.orchestra.run.smithi175.stdout:skipping the directory /tmp 2024-08-17T16:56:22.854 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:56:23.643 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:23.645 INFO:teuthology.orchestra.run.smithi175.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:56:23.736 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:56:23.736 INFO:teuthology.orchestra.run.smithi175.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 1/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 3/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 4/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 8/32 2024-08-17T16:56:24.915 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 9/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 12/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 13/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-17T16:56:24.916 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librados2-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-17T16:56:24.917 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 31/32 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout:Upgraded: 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout: librados2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:25.664 INFO:teuthology.orchestra.run.smithi175.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:56:25.665 INFO:teuthology.orchestra.run.smithi175.stdout: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.666 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:25.667 INFO:teuthology.orchestra.run.smithi175.stdout: re2-1:20211101-3.el9.x86_64 2024-08-17T16:56:25.667 INFO:teuthology.orchestra.run.smithi175.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:56:25.667 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:25.667 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:26.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-test 2024-08-17T16:56:26.570 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:25 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:26.695 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-test x86_64 2:18.2.4-719.gced649f8.el9 ceph 40 M 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout: libcephsqlite x86_64 2:18.2.4-719.gced649f8.el9 ceph 166 k 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:56:26.696 INFO:teuthology.orchestra.run.smithi175.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout:Install 4 Packages 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 40 M 2024-08-17T16:56:26.697 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 163 M 2024-08-17T16:56:26.698 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:26.744 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /sys 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /proc 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /mnt 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /var/tmp 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /home 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /root 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /tmp 2024-08-17T16:56:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:26.967 INFO:teuthology.orchestra.run.smithi175.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.0 MB/s | 304 kB 00:00 2024-08-17T16:56:27.009 INFO:teuthology.orchestra.run.smithi175.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-08-17T16:56:27.059 INFO:teuthology.orchestra.run.smithi175.stdout:(3/4): libcephsqlite-18.2.4-719.gced649f8.el9.x 695 kB/s | 166 kB 00:00 2024-08-17T16:56:27.519 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:56:27.546 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:56:27.546 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:27.546 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:56:27.546 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:56:27.546 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:56:27.547 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:27.548 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:56:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:56:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:28.134 INFO:teuthology.orchestra.run.smithi175.stdout:(4/4): ceph-test-18.2.4-719.gced649f8.el9.x86_6 30 MB/s | 40 MB 00:01 2024-08-17T16:56:28.136 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:28.137 INFO:teuthology.orchestra.run.smithi175.stdout:Total 28 MB/s | 40 MB 00:01 2024-08-17T16:56:28.164 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:28.184 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:28.184 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:28.501 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:28.501 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:28.755 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:28.755 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 1/32 2024-08-17T16:56:28.755 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2/32 2024-08-17T16:56:28.755 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 3/32 2024-08-17T16:56:28.755 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 4/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 8/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 9/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 12/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 13/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-17T16:56:28.756 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-17T16:56:28.757 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 31/32 2024-08-17T16:56:28.966 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:29.051 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:56:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:56:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout: librados2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:56:29.469 INFO:teuthology.orchestra.run.smithi064.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:56:29.470 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.471 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:29.472 INFO:teuthology.orchestra.run.smithi064.stdout: re2-1:20211101-3.el9.x86_64 2024-08-17T16:56:29.472 INFO:teuthology.orchestra.run.smithi064.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:56:29.472 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:29.472 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:29.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-test 2024-08-17T16:56:30.392 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:28 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:30.524 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test x86_64 2:18.2.4-719.gced649f8.el9 ceph 40 M 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite x86_64 2:18.2.4-719.gced649f8.el9 ceph 166 k 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:30.525 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:30.526 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:30.526 INFO:teuthology.orchestra.run.smithi064.stdout:Install 4 Packages 2024-08-17T16:56:30.526 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:30.526 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 40 M 2024-08-17T16:56:30.526 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 163 M 2024-08-17T16:56:30.527 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:30.911 INFO:teuthology.orchestra.run.smithi064.stdout:(1/4): libcephsqlite-18.2.4-719.gced649f8.el9.x 657 kB/s | 166 kB 00:00 2024-08-17T16:56:30.936 INFO:teuthology.orchestra.run.smithi064.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.1 MB/s | 304 kB 00:00 2024-08-17T16:56:31.061 INFO:teuthology.orchestra.run.smithi064.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 424 kB/s | 64 kB 00:00 2024-08-17T16:56:31.978 INFO:teuthology.orchestra.run.smithi064.stdout:(4/4): ceph-test-18.2.4-719.gced649f8.el9.x86_6 30 MB/s | 40 MB 00:01 2024-08-17T16:56:31.980 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:31.980 INFO:teuthology.orchestra.run.smithi064.stdout:Total 28 MB/s | 40 MB 00:01 2024-08-17T16:56:32.009 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:56:32.031 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:56:32.032 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:56:32.367 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:56:32.368 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:56:32.849 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:56:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:56:32.935 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:56:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:56:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:56:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:56:34.980 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:56:34.980 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 1/4 2024-08-17T16:56:34.980 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2/4 2024-08-17T16:56:34.980 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:56:35.249 INFO:teuthology.orchestra.run.smithi175.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:56:35.250 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:35.250 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:35.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph 2024-08-17T16:56:36.134 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:35 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:36.264 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:36.266 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:36.266 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:56:36.266 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:36.266 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph x86_64 2:18.2.4-719.gced649f8.el9 ceph 6.5 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mds x86_64 2:18.2.4-719.gced649f8.el9 ceph 2.1 M 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr x86_64 2:18.2.4-719.gced649f8.el9 ceph 1.5 M 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-modules-core noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 246 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mon x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.7 M 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-osd x86_64 2:18.2.4-719.gced649f8.el9 ceph 17 M 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:56:36.267 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:56:36.268 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout:Install 33 Packages 2024-08-17T16:56:36.269 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:36.270 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 29 M 2024-08-17T16:56:36.270 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 104 M 2024-08-17T16:56:36.270 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:36.954 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:56:37.039 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:56:37.136 INFO:teuthology.orchestra.run.smithi175.stdout:(1/33): ceph-18.2.4-719.gced649f8.el9.x86_64.rp 35 kB/s | 6.5 kB 00:00 2024-08-17T16:56:37.303 INFO:teuthology.orchestra.run.smithi175.stdout:(2/33): ceph-mgr-18.2.4-719.gced649f8.el9.x86_6 4.1 MB/s | 1.5 MB 00:00 2024-08-17T16:56:37.345 INFO:teuthology.orchestra.run.smithi175.stdout:(3/33): ceph-mds-18.2.4-719.gced649f8.el9.x86_6 5.3 MB/s | 2.1 MB 00:00 2024-08-17T16:56:37.429 INFO:teuthology.orchestra.run.smithi175.stdout:(4/33): ceph-mgr-modules-core-18.2.4-719.gced64 2.9 MB/s | 246 kB 00:00 2024-08-17T16:56:37.846 INFO:teuthology.orchestra.run.smithi175.stdout:(5/33): ceph-osd-18.2.4-719.gced649f8.el9.x86_6 30 MB/s | 17 MB 00:00 2024-08-17T16:56:37.921 INFO:teuthology.orchestra.run.smithi175.stdout:(6/33): ceph-mon-18.2.4-719.gced649f8.el9.x86_6 6.0 MB/s | 4.7 MB 00:00 2024-08-17T16:56:38.330 INFO:teuthology.orchestra.run.smithi175.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 281 kB/s | 253 kB 00:00 2024-08-17T16:56:38.663 INFO:teuthology.orchestra.run.smithi175.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 143 kB/s | 106 kB 00:00 2024-08-17T16:56:38.847 INFO:teuthology.orchestra.run.smithi175.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 1.2 MB/s | 1.2 MB 00:00 2024-08-17T16:56:39.035 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:56:39.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 1/4 2024-08-17T16:56:39.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2/4 2024-08-17T16:56:39.035 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:56:39.097 INFO:teuthology.orchestra.run.smithi175.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 176 kB/s | 135 kB 00:00 2024-08-17T16:56:39.338 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:56:39.338 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:39.338 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:56:39.338 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:39.338 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:39.339 INFO:teuthology.orchestra.run.smithi064.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:56:39.339 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:56:39.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:39.339 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:39.423 INFO:teuthology.orchestra.run.smithi175.stdout:(11/33): python3-requests-2.25.1-8.el9.noarch.r 165 kB/s | 125 kB 00:00 2024-08-17T16:56:39.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph 2024-08-17T16:56:39.690 INFO:teuthology.orchestra.run.smithi175.stdout:(12/33): python3-urllib3-1.26.5-6.el9.noarch.rp 255 kB/s | 215 kB 00:00 2024-08-17T16:56:39.865 INFO:teuthology.orchestra.run.smithi175.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 232 kB/s | 172 kB 00:00 2024-08-17T16:56:40.007 INFO:teuthology.orchestra.run.smithi175.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 60 kB/s | 35 kB 00:00 2024-08-17T16:56:40.074 INFO:teuthology.orchestra.run.smithi175.stdout:(15/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 663 kB/s | 43 kB 00:00 2024-08-17T16:56:40.115 INFO:teuthology.orchestra.run.smithi175.stdout:(16/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 4.0 MB/s | 172 kB 00:00 2024-08-17T16:56:40.174 INFO:teuthology.orchestra.run.smithi175.stdout:(17/33): python3-cherrypy-18.6.1-2.el9.noarch.r 11 MB/s | 358 kB 00:00 2024-08-17T16:56:40.197 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:38 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:40.199 INFO:teuthology.orchestra.run.smithi175.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 429 kB/s | 11 kB 00:00 2024-08-17T16:56:40.225 INFO:teuthology.orchestra.run.smithi175.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 712 kB/s | 18 kB 00:00 2024-08-17T16:56:40.250 INFO:teuthology.orchestra.run.smithi175.stdout:(20/33): python3-jaraco-collections-3.0.0-8.el9 965 kB/s | 23 kB 00:00 2024-08-17T16:56:40.275 INFO:teuthology.orchestra.run.smithi175.stdout:(21/33): python3-jaraco-functools-3.5.0-2.el9.n 778 kB/s | 19 kB 00:00 2024-08-17T16:56:40.292 INFO:teuthology.orchestra.run.smithi175.stdout:(22/33): python3-jaraco-text-3.2.0-6.el9.noarch 1.2 MB/s | 20 kB 00:00 2024-08-17T16:56:40.309 INFO:teuthology.orchestra.run.smithi175.stdout:(23/33): python3-pytz-2021.1-5.el9.noarch.rpm 82 kB/s | 51 kB 00:00 2024-08-17T16:56:40.326 INFO:teuthology.orchestra.run.smithi175.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.4 MB/s | 46 kB 00:00 2024-08-17T16:56:40.334 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:40.336 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph x86_64 2:18.2.4-719.gced649f8.el9 ceph 6.5 k 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds x86_64 2:18.2.4-719.gced649f8.el9 ceph 2.1 M 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr x86_64 2:18.2.4-719.gced649f8.el9 ceph 1.5 M 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 246 k 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.7 M 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd x86_64 2:18.2.4-719.gced649f8.el9 ceph 17 M 2024-08-17T16:56:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:56:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:56:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:Install 33 Packages 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 29 M 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 104 M 2024-08-17T16:56:40.340 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:40.393 INFO:teuthology.orchestra.run.smithi175.stdout:(25/33): python3-pecan-1.4.2-3.el9.noarch.rpm 6.4 MB/s | 272 kB 00:00 2024-08-17T16:56:40.410 INFO:teuthology.orchestra.run.smithi175.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 1.1 MB/s | 16 kB 00:00 2024-08-17T16:56:40.435 INFO:teuthology.orchestra.run.smithi175.stdout:(27/33): python3-more-itertools-8.12.0-2.el9.no 627 kB/s | 79 kB 00:00 2024-08-17T16:56:40.461 INFO:teuthology.orchestra.run.smithi175.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-08-17T16:56:40.477 INFO:teuthology.orchestra.run.smithi175.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 858 kB/s | 36 kB 00:00 2024-08-17T16:56:40.503 INFO:teuthology.orchestra.run.smithi175.stdout:(30/33): python3-webob-1.8.7-6.el9.noarch.rpm 5.4 MB/s | 230 kB 00:00 2024-08-17T16:56:40.520 INFO:teuthology.orchestra.run.smithi175.stdout:(31/33): python3-toml-0.10.2-6.el9.noarch.rpm 64 kB/s | 42 kB 00:00 2024-08-17T16:56:40.537 INFO:teuthology.orchestra.run.smithi175.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 611 kB/s | 20 kB 00:00 2024-08-17T16:56:40.562 INFO:teuthology.orchestra.run.smithi175.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.0 MB/s | 427 kB 00:00 2024-08-17T16:56:40.566 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:40.566 INFO:teuthology.orchestra.run.smithi175.stdout:Total 6.8 MB/s | 29 MB 00:04 2024-08-17T16:56:40.684 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:40.721 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:40.721 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:41.106 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:41.106 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:41.120 INFO:teuthology.orchestra.run.smithi064.stdout:(1/33): ceph-18.2.4-719.gced649f8.el9.x86_64.rp 34 kB/s | 6.5 kB 00:00 2024-08-17T16:56:41.296 INFO:teuthology.orchestra.run.smithi064.stdout:(2/33): ceph-mds-18.2.4-719.gced649f8.el9.x86_6 5.7 MB/s | 2.1 MB 00:00 2024-08-17T16:56:41.338 INFO:teuthology.orchestra.run.smithi064.stdout:(3/33): ceph-mgr-18.2.4-719.gced649f8.el9.x86_6 3.6 MB/s | 1.5 MB 00:00 2024-08-17T16:56:41.430 INFO:teuthology.orchestra.run.smithi064.stdout:(4/33): ceph-mgr-modules-core-18.2.4-719.gced64 2.6 MB/s | 246 kB 00:00 2024-08-17T16:56:41.588 INFO:teuthology.orchestra.run.smithi064.stdout:(5/33): ceph-mon-18.2.4-719.gced649f8.el9.x86_6 10 MB/s | 4.7 MB 00:00 2024-08-17T16:56:41.839 INFO:teuthology.orchestra.run.smithi064.stdout:(6/33): ceph-osd-18.2.4-719.gced649f8.el9.x86_6 30 MB/s | 17 MB 00:00 2024-08-17T16:56:42.014 INFO:teuthology.orchestra.run.smithi064.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 434 kB/s | 253 kB 00:00 2024-08-17T16:56:42.122 INFO:teuthology.orchestra.run.smithi064.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 1.2 MB/s | 135 kB 00:00 2024-08-17T16:56:42.223 INFO:teuthology.orchestra.run.smithi064.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 1.2 MB/s | 125 kB 00:00 2024-08-17T16:56:42.256 INFO:teuthology.orchestra.run.smithi064.stdout:(10/33): python3-ply-3.11-14.el9.noarch.rpm 255 kB/s | 106 kB 00:00 2024-08-17T16:56:42.273 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:42.340 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-17T16:56:42.372 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-17T16:56:42.405 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-17T16:56:42.456 INFO:teuthology.orchestra.run.smithi064.stdout:(11/33): python3-mako-1.1.4-6.el9.noarch.rpm 861 kB/s | 172 kB 00:00 2024-08-17T16:56:42.482 INFO:teuthology.orchestra.run.smithi064.stdout:(12/33): python3-urllib3-1.26.5-6.el9.noarch.rp 830 kB/s | 215 kB 00:00 2024-08-17T16:56:42.497 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-17T16:56:42.507 INFO:teuthology.orchestra.run.smithi064.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 693 kB/s | 35 kB 00:00 2024-08-17T16:56:42.557 INFO:teuthology.orchestra.run.smithi064.stdout:(14/33): python3-toml-0.10.2-6.el9.noarch.rpm 831 kB/s | 42 kB 00:00 2024-08-17T16:56:42.583 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-17T16:56:42.624 INFO:teuthology.orchestra.run.smithi064.stdout:(15/33): python3-cryptography-36.0.1-4.el9.x86_ 1.2 MB/s | 1.2 MB 00:01 2024-08-17T16:56:42.633 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-08-17T16:56:42.658 INFO:teuthology.orchestra.run.smithi064.stdout:(16/33): python3-pytz-2021.1-5.el9.noarch.rpm 290 kB/s | 51 kB 00:00 2024-08-17T16:56:42.676 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-17T16:56:42.700 INFO:teuthology.orchestra.run.smithi064.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 305 kB/s | 43 kB 00:00 2024-08-17T16:56:42.714 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-17T16:56:42.749 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-17T16:56:42.750 INFO:teuthology.orchestra.run.smithi064.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 511 kB/s | 11 kB 00:00 2024-08-17T16:56:42.775 INFO:teuthology.orchestra.run.smithi064.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-08-17T16:56:42.782 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-17T16:56:42.800 INFO:teuthology.orchestra.run.smithi064.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 370 kB/s | 18 kB 00:00 2024-08-17T16:56:42.823 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-17T16:56:42.834 INFO:teuthology.orchestra.run.smithi064.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 398 kB/s | 23 kB 00:00 2024-08-17T16:56:42.857 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-17T16:56:42.859 INFO:teuthology.orchestra.run.smithi064.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 331 kB/s | 19 kB 00:00 2024-08-17T16:56:42.885 INFO:teuthology.orchestra.run.smithi064.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-08-17T16:56:42.892 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-17T16:56:42.910 INFO:teuthology.orchestra.run.smithi064.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 922 kB/s | 46 kB 00:00 2024-08-17T16:56:42.935 INFO:teuthology.orchestra.run.smithi064.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T16:56:42.941 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-17T16:56:42.960 INFO:teuthology.orchestra.run.smithi064.stdout:(26/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-17T16:56:42.986 INFO:teuthology.orchestra.run.smithi064.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-08-17T16:56:43.011 INFO:teuthology.orchestra.run.smithi064.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-08-17T16:56:43.028 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-17T16:56:43.036 INFO:teuthology.orchestra.run.smithi064.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 711 kB/s | 36 kB 00:00 2024-08-17T16:56:43.061 INFO:teuthology.orchestra.run.smithi064.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-08-17T16:56:43.087 INFO:teuthology.orchestra.run.smithi064.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-08-17T16:56:43.112 INFO:teuthology.orchestra.run.smithi064.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-08-17T16:56:43.145 INFO:teuthology.orchestra.run.smithi064.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-08-17T16:56:43.149 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:43.149 INFO:teuthology.orchestra.run.smithi064.stdout:Total 10 MB/s | 29 MB 00:02 2024-08-17T16:56:43.175 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-17T16:56:43.255 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-17T16:56:43.267 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:56:43.311 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:56:43.311 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:56:43.312 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-17T16:56:43.348 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-17T16:56:43.421 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-17T16:56:43.485 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-17T16:56:43.678 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-17T16:56:43.711 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:56:43.712 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:56:43.727 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-17T16:56:43.806 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-17T16:56:43.894 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-17T16:56:43.934 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-17T16:56:44.051 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-17T16:56:44.210 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 28/33 2024-08-17T16:56:44.237 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:56:44.262 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:56:44.389 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:44.389 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:56:44.389 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:56:44.389 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:56:44.389 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:44.920 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:56:44.973 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-17T16:56:45.006 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-17T16:56:45.039 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-17T16:56:45.135 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-17T16:56:45.220 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-17T16:56:45.267 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-08-17T16:56:45.302 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-17T16:56:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-17T16:56:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-17T16:56:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-17T16:56:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-17T16:56:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-17T16:56:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-17T16:56:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-17T16:56:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-17T16:56:45.730 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:56:45.767 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-17T16:56:45.883 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-17T16:56:45.943 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-17T16:56:46.014 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-17T16:56:46.073 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-17T16:56:46.140 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-17T16:56:46.215 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:56:46.239 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:56:46.239 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:46.240 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:56:46.240 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:56:46.240 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:56:46.240 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:46.350 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-17T16:56:46.401 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-17T16:56:46.440 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:56:46.464 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:46.481 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-17T16:56:46.529 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:56:46.579 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-17T16:56:46.621 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-17T16:56:46.747 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-17T16:56:46.916 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 28/33 2024-08-17T16:56:46.943 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:56:46.970 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-2:18.2.4-719.gced649f8.el9.x86_64 1/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 3/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 4/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 5/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 6/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-17T16:56:48.211 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-17T16:56:48.212 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-17T16:56:48.213 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-17T16:56:48.214 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-17T16:56:48.214 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-17T16:56:48.214 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-08-17T16:56:48.214 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-17T16:56:48.486 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:56:48.517 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:49.012 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:56:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:56:49.038 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:49.039 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:56:49.039 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:56:49.039 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:56:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:49.099 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-17T16:56:49.099 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:49.099 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:56:49.100 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:56:49.101 INFO:teuthology.orchestra.run.smithi175.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:49.102 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:49.254 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:56:49.280 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:49.363 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:56:49.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-base 2024-08-17T16:56:50.060 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:50.144 INFO:teuthology.orchestra.run.smithi175.stdout:Package ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:56:50.185 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:50.186 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:56:50.186 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:50.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install cephadm 2024-08-17T16:56:50.798 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:50.924 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout: cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 224 k 2024-08-17T16:56:50.925 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 224 k 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 216 k 2024-08-17T16:56:50.926 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:51.072 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:56:51.072 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-2:18.2.4-719.gced649f8.el9.x86_64 1/33 2024-08-17T16:56:51.072 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2/33 2024-08-17T16:56:51.072 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 3/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 4/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 5/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 6/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-17T16:56:51.073 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-17T16:56:51.074 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-08-17T16:56:51.075 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-17T16:56:51.163 INFO:teuthology.orchestra.run.smithi175.stdout:cephadm-18.2.4-719.gced649f8.el9.noarch.rpm 947 kB/s | 224 kB 00:00 2024-08-17T16:56:51.164 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:51.164 INFO:teuthology.orchestra.run.smithi175.stdout:Total 940 kB/s | 224 kB 00:00 2024-08-17T16:56:51.164 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:51.169 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:51.169 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:51.172 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:51.172 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:51.245 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:51.683 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:51.774 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:52.090 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-17T16:56:52.090 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:56:52.091 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:56:52.092 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:52.093 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:52.116 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout: cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:52.361 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:52.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-base 2024-08-17T16:56:52.581 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:56:53.019 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:51 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:53.108 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:56:53.146 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:52 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:53.152 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:53.153 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:56:53.153 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:53.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install cephadm 2024-08-17T16:56:53.279 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:53.279 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:53.279 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repo Size 2024-08-17T16:56:53.279 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-719.gced649f8.el9 ceph 143 k 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 143 k 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 395 k 2024-08-17T16:56:53.280 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:53.511 INFO:teuthology.orchestra.run.smithi175.stdout:ceph-immutable-object-cache-18.2.4-719.gced649f 620 kB/s | 143 kB 00:00 2024-08-17T16:56:53.511 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:53.511 INFO:teuthology.orchestra.run.smithi175.stdout:Total 616 kB/s | 143 kB 00:00 2024-08-17T16:56:53.512 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:53.519 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:53.519 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:53.575 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:53.576 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:53.716 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:53.790 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:51 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:53.811 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:53.837 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:53.838 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:53.838 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:56:53.838 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:53.922 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 224 k 2024-08-17T16:56:53.924 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 224 k 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 216 k 2024-08-17T16:56:53.925 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:54.186 INFO:teuthology.orchestra.run.smithi064.stdout:cephadm-18.2.4-719.gced649f8.el9.noarch.rpm 856 kB/s | 224 kB 00:00 2024-08-17T16:56:54.187 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:54.187 INFO:teuthology.orchestra.run.smithi064.stdout:Total 851 kB/s | 224 kB 00:00 2024-08-17T16:56:54.187 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:56:54.192 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:56:54.193 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:56:54.195 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:56:54.196 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:56:54.270 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:56:54.725 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:54.746 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:54.746 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:54.746 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:54.746 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:54.746 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:54.747 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:54.809 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:54.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-mgr 2024-08-17T16:56:55.168 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:55.404 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:55.520 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:54 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:55.609 INFO:teuthology.orchestra.run.smithi175.stdout:Package ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:56:55.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:56:55.651 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:55.652 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:56:55.652 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:55.717 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:56:56.196 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:54 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:56.269 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:55 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:56.332 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-719.gced649f8.el9 ceph 143 k 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 143 k 2024-08-17T16:56:56.333 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 395 k 2024-08-17T16:56:56.334 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:56.396 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-dashboard noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 3.5 M 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-grafana-dashboards noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 24 k 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-prometheus-alerts noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 16 k 2024-08-17T16:56:56.397 INFO:teuthology.orchestra.run.smithi175.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout:Install 5 Packages 2024-08-17T16:56:56.398 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:56.399 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 3.8 M 2024-08-17T16:56:56.399 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 87 M 2024-08-17T16:56:56.399 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:56:56.551 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-immutable-object-cache-18.2.4-719.gced649f 655 kB/s | 143 kB 00:00 2024-08-17T16:56:56.552 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:56.552 INFO:teuthology.orchestra.run.smithi064.stdout:Total 649 kB/s | 143 kB 00:00 2024-08-17T16:56:56.552 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:56:56.560 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:56:56.560 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:56:56.620 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:56:56.620 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:56:56.664 INFO:teuthology.orchestra.run.smithi175.stdout:(1/5): ceph-grafana-dashboards-18.2.4-719.gced6 130 kB/s | 24 kB 00:00 2024-08-17T16:56:56.714 INFO:teuthology.orchestra.run.smithi175.stdout:(2/5): ceph-prometheus-alerts-18.2.4-719.gced64 69 kB/s | 16 kB 00:00 2024-08-17T16:56:56.770 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:56:56.797 INFO:teuthology.orchestra.run.smithi175.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 230 kB/s | 31 kB 00:00 2024-08-17T16:56:56.840 INFO:teuthology.orchestra.run.smithi175.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 1.5 MB/s | 188 kB 00:00 2024-08-17T16:56:56.852 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:56.878 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:56.878 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:56:56.878 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:56:56.878 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:56.940 INFO:teuthology.orchestra.run.smithi175.stdout:(5/5): ceph-mgr-dashboard-18.2.4-719.gced649f8. 7.6 MB/s | 3.5 MB 00:00 2024-08-17T16:56:56.944 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:56:56.944 INFO:teuthology.orchestra.run.smithi175.stdout:Total 6.9 MB/s | 3.8 MB 00:00 2024-08-17T16:56:56.944 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:56:56.951 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:56:56.951 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:56:57.012 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:56:57.012 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:56:57.256 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:56:57.337 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-17T16:56:57.369 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-17T16:56:57.398 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:56:57.828 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:56:57.828 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:57.828 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:56:57.829 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:56:57.829 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:57.829 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:58.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr 2024-08-17T16:56:58.441 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 4/5 2024-08-17T16:56:58.477 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:56:58.627 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:56 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:58.723 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:56:58.767 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:58.768 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:56:58.768 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:56:58.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:56:59.393 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:57 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:56:59.525 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:56:59.525 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 3.5 M 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 24 k 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 16 k 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout:Install 5 Packages 2024-08-17T16:56:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:56:59.527 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 3.8 M 2024-08-17T16:56:59.527 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 87 M 2024-08-17T16:56:59.527 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:56:59.619 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:56:59.619 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 1/5 2024-08-17T16:56:59.619 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2/5 2024-08-17T16:56:59.619 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:56:59.619 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-17T16:56:59.805 INFO:teuthology.orchestra.run.smithi064.stdout:(1/5): ceph-grafana-dashboards-18.2.4-719.gced6 123 kB/s | 24 kB 00:00 2024-08-17T16:56:59.830 INFO:teuthology.orchestra.run.smithi064.stdout:(2/5): ceph-prometheus-alerts-18.2.4-719.gced64 74 kB/s | 16 kB 00:00 2024-08-17T16:56:59.876 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:56:59.877 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:56:59.913 INFO:teuthology.orchestra.run.smithi064.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 283 kB/s | 31 kB 00:00 2024-08-17T16:56:59.947 INFO:teuthology.orchestra.run.smithi064.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 1.6 MB/s | 188 kB 00:00 2024-08-17T16:57:00.072 INFO:teuthology.orchestra.run.smithi064.stdout:(5/5): ceph-mgr-dashboard-18.2.4-719.gced649f8. 7.6 MB/s | 3.5 MB 00:00 2024-08-17T16:57:00.075 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:00.075 INFO:teuthology.orchestra.run.smithi064.stdout:Total 6.9 MB/s | 3.8 MB 00:00 2024-08-17T16:57:00.076 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:00.083 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:00.083 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:00.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:57:00.148 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:00.148 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:00.402 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:00.479 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-17T16:57:00.511 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-17T16:57:00.540 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:57:00.659 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:00:59 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:00.787 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:00.788 INFO:teuthology.orchestra.run.smithi175.stdout:====================================================================================== 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout:====================================================================================== 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 7.4 M 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:57:00.789 INFO:teuthology.orchestra.run.smithi175.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout:====================================================================================== 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout:Install 12 Packages 2024-08-17T16:57:00.790 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:00.791 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 38 M 2024-08-17T16:57:00.791 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 209 M 2024-08-17T16:57:00.791 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:01.063 INFO:teuthology.orchestra.run.smithi175.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.1 MB/s | 195 kB 00:00 2024-08-17T16:57:01.130 INFO:teuthology.orchestra.run.smithi175.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 491 kB/s | 33 kB 00:00 2024-08-17T16:57:01.172 INFO:teuthology.orchestra.run.smithi175.stdout:(3/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.8 MB/s | 803 kB 00:00 2024-08-17T16:57:01.255 INFO:teuthology.orchestra.run.smithi175.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 219 kB/s | 18 kB 00:00 2024-08-17T16:57:01.339 INFO:teuthology.orchestra.run.smithi175.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 14 MB/s | 3.0 MB 00:00 2024-08-17T16:57:01.364 INFO:teuthology.orchestra.run.smithi175.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 324 kB/s | 35 kB 00:00 2024-08-17T16:57:01.456 INFO:teuthology.orchestra.run.smithi175.stdout:(7/12): ceph-mgr-diskprediction-local-18.2.4-71 13 MB/s | 7.4 MB 00:00 2024-08-17T16:57:01.481 INFO:teuthology.orchestra.run.smithi175.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 2.0 MB/s | 244 kB 00:00 2024-08-17T16:57:01.548 INFO:teuthology.orchestra.run.smithi175.stdout:(9/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 23 MB/s | 4.7 MB 00:00 2024-08-17T16:57:01.582 INFO:teuthology.orchestra.run.smithi175.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.4 MB/s | 250 kB 00:00 2024-08-17T16:57:01.654 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 4/5 2024-08-17T16:57:01.701 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:57:01.874 INFO:teuthology.orchestra.run.smithi175.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 50 MB/s | 16 MB 00:00 2024-08-17T16:57:02.082 INFO:teuthology.orchestra.run.smithi175.stdout:(12/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 8.1 MB/s | 5.1 MB 00:00 2024-08-17T16:57:02.083 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:02.083 INFO:teuthology.orchestra.run.smithi175.stdout:Total 29 MB/s | 38 MB 00:01 2024-08-17T16:57:02.435 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:02.462 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:02.462 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:57:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 1/5 2024-08-17T16:57:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2/5 2024-08-17T16:57:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:57:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-17T16:57:02.961 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:02.961 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-17T16:57:02.961 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:02.962 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:03.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:57:03.717 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:03.722 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:01 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:03.791 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:57:03.834 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:57:03.855 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout:====================================================================================== 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout:====================================================================================== 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 7.4 M 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:57:03.857 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout:====================================================================================== 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout:Install 12 Packages 2024-08-17T16:57:03.858 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:03.859 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 38 M 2024-08-17T16:57:03.859 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 209 M 2024-08-17T16:57:03.859 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:03.932 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:57:03.965 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:57:04.141 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:57:04.172 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:57:04.248 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:57:04.587 INFO:teuthology.orchestra.run.smithi064.stdout:(1/12): ceph-mgr-diskprediction-local-18.2.4-71 16 MB/s | 7.4 MB 00:00 2024-08-17T16:57:04.629 INFO:teuthology.orchestra.run.smithi064.stdout:(2/12): libquadmath-11.4.1-3.el9.x86_64.rpm 381 kB/s | 195 kB 00:00 2024-08-17T16:57:04.713 INFO:teuthology.orchestra.run.smithi064.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 261 kB/s | 33 kB 00:00 2024-08-17T16:57:04.763 INFO:teuthology.orchestra.run.smithi064.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 364 kB/s | 18 kB 00:00 2024-08-17T16:57:04.847 INFO:teuthology.orchestra.run.smithi064.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 422 kB/s | 35 kB 00:00 2024-08-17T16:57:04.905 INFO:teuthology.orchestra.run.smithi064.stdout:(6/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.0 MB/s | 803 kB 00:00 2024-08-17T16:57:04.919 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:57:05.035 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:57:05.039 INFO:teuthology.orchestra.run.smithi064.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 7.3 MB/s | 3.0 MB 00:00 2024-08-17T16:57:05.173 INFO:teuthology.orchestra.run.smithi064.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 15 MB/s | 4.7 MB 00:00 2024-08-17T16:57:05.198 INFO:teuthology.orchestra.run.smithi064.stdout:(9/12): python3-devel-3.9.18-3.el9.x86_64.rpm 835 kB/s | 244 kB 00:00 2024-08-17T16:57:05.223 INFO:teuthology.orchestra.run.smithi064.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 4.9 MB/s | 250 kB 00:00 2024-08-17T16:57:05.290 INFO:teuthology.orchestra.run.smithi064.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 20 MB/s | 5.1 MB 00:00 2024-08-17T16:57:06.314 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:57:07.335 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:57:07.362 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:57:07.432 INFO:teuthology.orchestra.run.smithi064.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 7.3 MB/s | 16 MB 00:02 2024-08-17T16:57:07.432 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:07.432 INFO:teuthology.orchestra.run.smithi064.stdout:Total 11 MB/s | 38 MB 00:03 2024-08-17T16:57:07.795 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:07.823 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:07.823 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:08.360 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:08.360 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:57:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:57:09.395 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:57:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:57:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:57:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:57:09.736 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 1/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:57:09.791 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:57:09.792 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:57:10.203 INFO:teuthology.orchestra.run.smithi175.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:10.204 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:10.447 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:57:10.574 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:57:10.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:57:11.185 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:10 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:11.313 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-rook noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 49 k 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:57:11.315 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout:Install 13 Packages 2024-08-17T16:57:11.316 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:11.317 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 2.2 M 2024-08-17T16:57:11.317 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 26 M 2024-08-17T16:57:11.317 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:11.561 INFO:teuthology.orchestra.run.smithi175.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 186 kB/s | 19 kB 00:00 2024-08-17T16:57:11.586 INFO:teuthology.orchestra.run.smithi175.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 200 kB/s | 26 kB 00:00 2024-08-17T16:57:11.645 INFO:teuthology.orchestra.run.smithi175.stdout:(3/13): ceph-mgr-rook-18.2.4-719.gced649f8.el9. 264 kB/s | 49 kB 00:00 2024-08-17T16:57:11.721 INFO:teuthology.orchestra.run.smithi175.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-08-17T16:57:11.746 INFO:teuthology.orchestra.run.smithi175.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.2 MB/s | 222 kB 00:00 2024-08-17T16:57:11.796 INFO:teuthology.orchestra.run.smithi175.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 714 kB/s | 54 kB 00:00 2024-08-17T16:57:11.846 INFO:teuthology.orchestra.run.smithi175.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 321 kB/s | 32 kB 00:00 2024-08-17T16:57:11.871 INFO:teuthology.orchestra.run.smithi175.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.2 MB/s | 279 kB 00:00 2024-08-17T16:57:11.896 INFO:teuthology.orchestra.run.smithi175.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 145 kB/s | 14 kB 00:00 2024-08-17T16:57:11.905 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:57:11.946 INFO:teuthology.orchestra.run.smithi175.stdout:(10/13): python3-google-auth-2.33.0-1.el9.noarc 2.2 MB/s | 220 kB 00:00 2024-08-17T16:57:11.971 INFO:teuthology.orchestra.run.smithi175.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-08-17T16:57:12.004 INFO:teuthology.orchestra.run.smithi175.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.6 MB/s | 90 kB 00:00 2024-08-17T16:57:12.055 INFO:teuthology.orchestra.run.smithi175.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 5.6 MB/s | 1.0 MB 00:00 2024-08-17T16:57:12.057 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:12.057 INFO:teuthology.orchestra.run.smithi175.stdout:Total 3.0 MB/s | 2.2 MB 00:00 2024-08-17T16:57:12.121 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:12.136 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:12.136 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:12.223 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:12.223 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:12.729 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:12.908 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:57:12.972 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:57:13.000 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:57:13.029 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:57:13.039 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:57:13.078 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:57:13.119 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:57:13.221 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:57:13.331 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:57:13.379 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:57:14.094 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:57:14.145 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:57:14.178 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:57:14.226 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:57:14.249 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 1/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:57:15.060 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:57:15.061 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:57:15.061 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:57:15.063 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:57:15.064 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:57:15.064 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:57:15.393 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:15.394 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 1/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:57:15.473 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:57:15.474 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:57:15.474 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:57:15.474 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:57:15.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:57:15.863 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:57:15.863 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:15.863 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:15.863 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:15.863 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:15.864 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:16.230 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:15 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:16.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:57:16.358 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:16.358 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:16.358 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 139 k 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:16.359 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:16.360 INFO:teuthology.orchestra.run.smithi175.stdout:Install 6 Packages 2024-08-17T16:57:16.360 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:16.360 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 7.0 M 2024-08-17T16:57:16.360 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 33 M 2024-08-17T16:57:16.360 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:16.818 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:14 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:16.952 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 49 k 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:57:16.954 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout:Install 13 Packages 2024-08-17T16:57:16.955 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:16.956 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 2.2 M 2024-08-17T16:57:16.956 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 26 M 2024-08-17T16:57:16.956 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:17.248 INFO:teuthology.orchestra.run.smithi175.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-08-17T16:57:17.298 INFO:teuthology.orchestra.run.smithi175.stdout:(2/6): ceph-mgr-cephadm-18.2.4-719.gced649f8.el 606 kB/s | 139 kB 00:00 2024-08-17T16:57:17.357 INFO:teuthology.orchestra.run.smithi175.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.9 MB/s | 548 kB 00:00 2024-08-17T16:57:17.390 INFO:teuthology.orchestra.run.smithi175.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.4 MB/s | 74 kB 00:00 2024-08-17T16:57:17.415 INFO:teuthology.orchestra.run.smithi175.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 503 kB/s | 58 kB 00:00 2024-08-17T16:57:17.532 INFO:teuthology.orchestra.run.smithi175.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 13 MB/s | 6.0 MB 00:00 2024-08-17T16:57:17.535 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:17.535 INFO:teuthology.orchestra.run.smithi175.stdout:Total 6.0 MB/s | 7.0 MB 00:01 2024-08-17T16:57:17.607 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:17.618 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:17.618 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:17.716 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:17.717 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:17.771 INFO:teuthology.orchestra.run.smithi064.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 173 kB/s | 19 kB 00:00 2024-08-17T16:57:17.797 INFO:teuthology.orchestra.run.smithi064.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 188 kB/s | 26 kB 00:00 2024-08-17T16:57:17.856 INFO:teuthology.orchestra.run.smithi064.stdout:(3/13): ceph-mgr-rook-18.2.4-719.gced649f8.el9. 253 kB/s | 49 kB 00:00 2024-08-17T16:57:17.931 INFO:teuthology.orchestra.run.smithi064.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-08-17T16:57:17.956 INFO:teuthology.orchestra.run.smithi064.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.2 MB/s | 222 kB 00:00 2024-08-17T16:57:18.014 INFO:teuthology.orchestra.run.smithi064.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 553 kB/s | 32 kB 00:00 2024-08-17T16:57:18.028 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:18.040 INFO:teuthology.orchestra.run.smithi064.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 565 kB/s | 14 kB 00:00 2024-08-17T16:57:18.073 INFO:teuthology.orchestra.run.smithi064.stdout:(8/13): python3-requests-oauthlib-1.3.0-12.el9. 377 kB/s | 54 kB 00:00 2024-08-17T16:57:18.098 INFO:teuthology.orchestra.run.smithi064.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.1 MB/s | 279 kB 00:00 2024-08-17T16:57:18.182 INFO:teuthology.orchestra.run.smithi064.stdout:(10/13): python3-google-auth-2.33.0-1.el9.noarc 1.5 MB/s | 220 kB 00:00 2024-08-17T16:57:18.206 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:57:18.215 INFO:teuthology.orchestra.run.smithi064.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.1 MB/s | 90 kB 00:00 2024-08-17T16:57:18.240 INFO:teuthology.orchestra.run.smithi064.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 507 kB/s | 59 kB 00:00 2024-08-17T16:57:18.259 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:57:18.283 INFO:teuthology.orchestra.run.smithi064.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 4.9 MB/s | 1.0 MB 00:00 2024-08-17T16:57:18.285 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:18.285 INFO:teuthology.orchestra.run.smithi064.stdout:Total 1.7 MB/s | 2.2 MB 00:01 2024-08-17T16:57:18.350 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:18.366 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:18.366 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:18.463 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:18.463 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:18.774 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:57:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:57:18.925 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:57:18.955 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:57:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:19.172 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:57:19.236 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:57:19.312 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:57:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:57:19.399 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:57:19.506 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:57:19.633 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:57:19.684 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:57:19.735 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:57:19.736 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/6 2024-08-17T16:57:19.736 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:57:19.736 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:57:19.736 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:57:19.736 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:57:20.006 INFO:teuthology.orchestra.run.smithi175.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:57:20.007 INFO:teuthology.orchestra.run.smithi175.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:57:20.007 INFO:teuthology.orchestra.run.smithi175.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:57:20.007 INFO:teuthology.orchestra.run.smithi175.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:57:20.007 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:20.007 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:20.309 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install ceph-fuse 2024-08-17T16:57:20.447 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:57:20.499 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:57:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:57:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:57:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:57:20.864 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:19 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:20.993 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:20.994 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:20.994 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repository Size 2024-08-17T16:57:20.994 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:20.994 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 852 k 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Install 2 Packages 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 931 k 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 2.7 M 2024-08-17T16:57:20.995 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:21.209 INFO:teuthology.orchestra.run.smithi175.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 525 kB/s | 80 kB 00:00 2024-08-17T16:57:21.368 INFO:teuthology.orchestra.run.smithi175.stdout:(2/2): ceph-fuse-18.2.4-719.gced649f8.el9.x86_6 2.7 MB/s | 852 kB 00:00 2024-08-17T16:57:21.368 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:21.368 INFO:teuthology.orchestra.run.smithi175.stdout:Total 2.4 MB/s | 931 kB 00:00 2024-08-17T16:57:21.380 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:57:21.389 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 1/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:57:21.390 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:57:21.401 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:21.401 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:21.480 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:21.480 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:21.660 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:21.726 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:57:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:21.728 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:21.791 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:57:21.867 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:57:21.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:57:22.548 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:20 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:22.558 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:57:22.558 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/2 2024-08-17T16:57:22.681 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 139 k 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:57:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 7.0 M 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 33 M 2024-08-17T16:57:22.684 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:22.823 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:57:22.824 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:22.824 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:22.824 INFO:teuthology.orchestra.run.smithi175.stdout: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:57:22.824 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:22.824 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:23.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install librados-devel 2024-08-17T16:57:23.569 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-08-17T16:57:23.628 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): ceph-mgr-cephadm-18.2.4-719.gced649f8.el 584 kB/s | 139 kB 00:00 2024-08-17T16:57:23.644 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:22 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:23.686 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.6 MB/s | 548 kB 00:00 2024-08-17T16:57:23.711 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 3.3 MB/s | 74 kB 00:00 2024-08-17T16:57:23.737 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 531 kB/s | 58 kB 00:00 2024-08-17T16:57:23.777 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repo Size 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout: librados-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 127 k 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:57:23.778 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:23.779 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 127 k 2024-08-17T16:57:23.779 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 456 k 2024-08-17T16:57:23.779 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:23.829 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-08-17T16:57:23.832 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:23.832 INFO:teuthology.orchestra.run.smithi064.stdout:Total 6.1 MB/s | 7.0 MB 00:01 2024-08-17T16:57:23.905 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:23.917 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:23.917 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:24.010 INFO:teuthology.orchestra.run.smithi175.stdout:librados-devel-18.2.4-719.gced649f8.el9.x86_64. 548 kB/s | 127 kB 00:00 2024-08-17T16:57:24.011 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:24.011 INFO:teuthology.orchestra.run.smithi175.stdout:Total 545 kB/s | 127 kB 00:00 2024-08-17T16:57:24.011 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:24.016 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:24.016 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:24.021 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:24.023 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:24.078 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:24.079 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:24.228 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:24.334 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:24.335 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:24.522 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:57:24.573 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:57:24.772 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:25.023 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:25.023 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:25.023 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:25.024 INFO:teuthology.orchestra.run.smithi175.stdout: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:25.024 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:25.024 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:25.149 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:57:25.237 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:57:25.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install libcephfs2 2024-08-17T16:57:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:57:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:57:25.839 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:24 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:25.938 INFO:teuthology.orchestra.run.smithi175.stdout:Package libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:25.982 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:25.983 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:25.984 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:26.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install libcephfs-devel 2024-08-17T16:57:26.095 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:57:26.095 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/6 2024-08-17T16:57:26.095 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:57:26.095 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:57:26.096 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:57:26.096 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:26.367 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:26.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-fuse 2024-08-17T16:57:26.606 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:26.735 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:26.735 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:26.735 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repo Size 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout: libcephfs-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 31 k 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:57:26.736 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:26.737 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 31 k 2024-08-17T16:57:26.737 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 139 k 2024-08-17T16:57:26.737 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:26.899 INFO:teuthology.orchestra.run.smithi175.stdout:libcephfs-devel-18.2.4-719.gced649f8.el9.x86_64 192 kB/s | 31 kB 00:00 2024-08-17T16:57:26.899 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:26.899 INFO:teuthology.orchestra.run.smithi175.stdout:Total 190 kB/s | 31 kB 00:00 2024-08-17T16:57:26.900 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:26.903 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:26.903 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:26.918 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:26.919 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:26.999 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:27.113 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:27.177 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:27.309 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 852 k 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:57:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:Install 2 Packages 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 931 k 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 2.7 M 2024-08-17T16:57:27.311 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:27.532 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:27.723 INFO:teuthology.orchestra.run.smithi064.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 583 kB/s | 80 kB 00:00 2024-08-17T16:57:27.804 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:27.804 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:27.804 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:27.805 INFO:teuthology.orchestra.run.smithi175.stdout: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:27.805 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:27.805 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:27.881 INFO:teuthology.orchestra.run.smithi064.stdout:(2/2): ceph-fuse-18.2.4-719.gced649f8.el9.x86_6 2.8 MB/s | 852 kB 00:00 2024-08-17T16:57:27.882 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:27.882 INFO:teuthology.orchestra.run.smithi064.stdout:Total 1.6 MB/s | 931 kB 00:00 2024-08-17T16:57:27.894 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:27.917 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:27.918 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:27.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install librados2 2024-08-17T16:57:28.002 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:28.002 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:28.334 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:57:28.417 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:57:28.548 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:27 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:28.634 INFO:teuthology.orchestra.run.smithi175.stdout:Package librados2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:28.676 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:28.677 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:28.677 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:28.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install librbd1 2024-08-17T16:57:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:57:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/2 2024-08-17T16:57:29.307 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:29.369 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:57:29.369 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:29.369 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:29.369 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:57:29.369 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:29.370 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:29.393 INFO:teuthology.orchestra.run.smithi175.stdout:Package librbd1-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:29.435 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:29.436 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:29.436 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:29.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install python3-rados 2024-08-17T16:57:29.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados-devel 2024-08-17T16:57:30.069 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:29 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:30.155 INFO:teuthology.orchestra.run.smithi175.stdout:Package python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:30.197 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:30.198 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:30.198 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:30.206 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:30.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install python3-rgw 2024-08-17T16:57:30.344 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:30.345 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 127 k 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:30.346 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:57:30.347 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:30.347 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 127 k 2024-08-17T16:57:30.347 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 456 k 2024-08-17T16:57:30.347 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:30.558 INFO:teuthology.orchestra.run.smithi064.stdout:librados-devel-18.2.4-719.gced649f8.el9.x86_64. 598 kB/s | 127 kB 00:00 2024-08-17T16:57:30.559 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:30.559 INFO:teuthology.orchestra.run.smithi064.stdout:Total 595 kB/s | 127 kB 00:00 2024-08-17T16:57:30.559 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:30.564 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:30.565 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:30.824 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:29 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:30.900 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:30.910 INFO:teuthology.orchestra.run.smithi175.stdout:Package python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:30.952 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:30.953 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:30.953 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:31.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install python3-cephfs 2024-08-17T16:57:31.319 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:31.585 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:31.588 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:31.589 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:31.589 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:31.589 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:31.589 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:31.589 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:31.671 INFO:teuthology.orchestra.run.smithi175.stdout:Package python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:31.713 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:31.714 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:31.714 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:31.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install python3-rbd 2024-08-17T16:57:31.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs2 2024-08-17T16:57:32.342 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:32.428 INFO:teuthology.orchestra.run.smithi175.stdout:Package python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:32.435 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:32.470 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:32.471 INFO:teuthology.orchestra.run.smithi175.stdout:Nothing to do. 2024-08-17T16:57:32.471 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:32.524 INFO:teuthology.orchestra.run.smithi064.stdout:Package libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:32.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install rbd-fuse 2024-08-17T16:57:32.569 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:32.570 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:32.570 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:32.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs-devel 2024-08-17T16:57:33.104 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:33.213 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:33.231 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 87 k 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:57:33.232 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:33.233 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 87 k 2024-08-17T16:57:33.233 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 227 k 2024-08-17T16:57:33.233 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:33.347 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 31 k 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 31 k 2024-08-17T16:57:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 139 k 2024-08-17T16:57:33.349 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:33.435 INFO:teuthology.orchestra.run.smithi175.stdout:rbd-fuse-18.2.4-719.gced649f8.el9.x86_64.rpm 429 kB/s | 87 kB 00:00 2024-08-17T16:57:33.435 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:33.435 INFO:teuthology.orchestra.run.smithi175.stdout:Total 427 kB/s | 87 kB 00:00 2024-08-17T16:57:33.435 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:33.442 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:33.442 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:33.505 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:33.505 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:33.532 INFO:teuthology.orchestra.run.smithi064.stdout:libcephfs-devel-18.2.4-719.gced649f8.el9.x86_64 169 kB/s | 31 kB 00:00 2024-08-17T16:57:33.532 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:33.533 INFO:teuthology.orchestra.run.smithi064.stdout:Total 168 kB/s | 31 kB 00:00 2024-08-17T16:57:33.533 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:33.537 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:33.537 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:33.549 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:33.550 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:33.634 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:33.651 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:33.748 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:33.757 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:34.095 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:34.139 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:34.337 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:34.338 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:34.338 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:34.338 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:34.338 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:34.338 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:34.374 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:34.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados2 2024-08-17T16:57:34.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install rbd-mirror 2024-08-17T16:57:35.117 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:35.162 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:35.205 INFO:teuthology.orchestra.run.smithi064.stdout:Package librados2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:35.249 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:35.250 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:35.251 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout: Package Arch Version Repo Size 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:35.294 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-mirror x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 3.0 M 2024-08-17T16:57:35.295 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 12 M 2024-08-17T16:57:35.296 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:35.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librbd1 2024-08-17T16:57:35.623 INFO:teuthology.orchestra.run.smithi175.stdout:rbd-mirror-18.2.4-719.gced649f8.el9.x86_64.rpm 9.1 MB/s | 3.0 MB 00:00 2024-08-17T16:57:35.624 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:35.624 INFO:teuthology.orchestra.run.smithi175.stdout:Total 9.1 MB/s | 3.0 MB 00:00 2024-08-17T16:57:35.624 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:35.633 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:35.633 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:35.713 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:35.714 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:35.885 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:35.973 INFO:teuthology.orchestra.run.smithi064.stdout:Package librbd1-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:36.019 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:36.020 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:36.020 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:36.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rados 2024-08-17T16:57:36.126 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:36.193 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:36.218 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:36.219 INFO:teuthology.orchestra.run.smithi175.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:57:36.219 INFO:teuthology.orchestra.run.smithi175.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:57:36.219 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:57:36.219 INFO:teuthology.orchestra.run.smithi175.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:57:36.219 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:36.653 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:36.741 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:36.786 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:36.787 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:36.788 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:36.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rgw 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:37.320 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:37.425 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:35 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:37.514 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:37.558 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:37.559 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:37.560 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:37.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum -y install rbd-nbd 2024-08-17T16:57:37.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-cephfs 2024-08-17T16:57:38.146 INFO:teuthology.orchestra.run.smithi175.stdout:Last metadata expiration check: 0:01:37 ago on Sat 17 Aug 2024 04:56:01 PM UTC. 2024-08-17T16:57:38.195 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:36 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:38.280 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-08-17T16:57:38.280 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-nbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 172 k 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Install 1 Package 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 172 k 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 494 k 2024-08-17T16:57:38.281 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-08-17T16:57:38.285 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:38.329 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:38.330 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:38.331 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:38.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rbd 2024-08-17T16:57:38.507 INFO:teuthology.orchestra.run.smithi175.stdout:rbd-nbd-18.2.4-719.gced649f8.el9.x86_64.rpm 764 kB/s | 172 kB 00:00 2024-08-17T16:57:38.507 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:38.508 INFO:teuthology.orchestra.run.smithi175.stdout:Total 760 kB/s | 172 kB 00:00 2024-08-17T16:57:38.508 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-08-17T16:57:38.515 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-08-17T16:57:38.515 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-08-17T16:57:38.577 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-08-17T16:57:38.578 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-08-17T16:57:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-08-17T16:57:38.829 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:38.963 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:36 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:39.051 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:57:39.096 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:39.097 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2024-08-17T16:57:39.097 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:39.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-fuse 2024-08-17T16:57:39.264 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T16:57:39.530 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-08-17T16:57:39.731 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:37 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:39.770 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:39.865 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 87 k 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:39.867 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:57:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:39.867 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 87 k 2024-08-17T16:57:39.867 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 227 k 2024-08-17T16:57:39.867 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:40.069 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-fuse-18.2.4-719.gced649f8.el9.x86_64.rpm 430 kB/s | 87 kB 00:00 2024-08-17T16:57:40.069 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:40.070 INFO:teuthology.orchestra.run.smithi064.stdout:Total 427 kB/s | 87 kB 00:00 2024-08-17T16:57:40.070 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:40.077 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:40.077 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:40.145 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:40.145 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:40.403 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:40.744 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:40.993 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:41.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-mirror 2024-08-17T16:57:41.779 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:39 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:41.914 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:41.915 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:41.915 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2024-08-17T16:57:41.915 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 3.0 M 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 12 M 2024-08-17T16:57:41.916 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:42.265 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-mirror-18.2.4-719.gced649f8.el9.x86_64.rpm 8.6 MB/s | 3.0 MB 00:00 2024-08-17T16:57:42.266 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:42.266 INFO:teuthology.orchestra.run.smithi064.stdout:Total 8.5 MB/s | 3.0 MB 00:00 2024-08-17T16:57:42.266 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:42.276 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:42.276 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:42.362 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:42.362 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:42.803 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:42.866 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:42.892 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:42.892 INFO:teuthology.orchestra.run.smithi064.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:57:42.892 INFO:teuthology.orchestra.run.smithi064.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:57:42.893 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:57:42.893 INFO:teuthology.orchestra.run.smithi064.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:57:42.893 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:44.013 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:44.014 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:44.014 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:44.014 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:44.014 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:44.014 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:44.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-nbd 2024-08-17T16:57:44.795 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:42 ago on Sat 17 Aug 2024 04:56:02 PM UTC. 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2024-08-17T16:57:44.929 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 172 k 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 172 k 2024-08-17T16:57:44.930 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 494 k 2024-08-17T16:57:44.931 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2024-08-17T16:57:45.165 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-nbd-18.2.4-719.gced649f8.el9.x86_64.rpm 735 kB/s | 172 kB 00:00 2024-08-17T16:57:45.165 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:57:45.166 INFO:teuthology.orchestra.run.smithi064.stdout:Total 730 kB/s | 172 kB 00:00 2024-08-17T16:57:45.166 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2024-08-17T16:57:45.175 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2024-08-17T16:57:45.175 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2024-08-17T16:57:45.242 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2024-08-17T16:57:45.242 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2024-08-17T16:57:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2024-08-17T16:57:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:45.903 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:46.155 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:57:46.155 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:46.155 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2024-08-17T16:57:46.155 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:57:46.155 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:57:46.156 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2024-08-17T16:57:46.390 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:46.391 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:57:46.391 INFO:teuthology.packaging:ref: None 2024-08-17T16:57:46.391 INFO:teuthology.packaging:tag: None 2024-08-17T16:57:46.391 INFO:teuthology.packaging:branch: reef 2024-08-17T16:57:46.391 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:57:46.391 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:57:46.672 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:57:46.697 INFO:teuthology.orchestra.run.smithi064.stdout:18.2.4-719.gced649f8.el9 2024-08-17T16:57:46.697 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-719.gced649f8.el9 2024-08-17T16:57:46.697 INFO:teuthology.task.install:The correct ceph version 18.2.4-719.gced649f8 is installed. 2024-08-17T16:57:46.699 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:57:46.699 INFO:teuthology.packaging:ref: None 2024-08-17T16:57:46.699 INFO:teuthology.packaging:tag: None 2024-08-17T16:57:46.699 INFO:teuthology.packaging:branch: reef 2024-08-17T16:57:46.699 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:57:46.700 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:57:46.973 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:57:46.998 INFO:teuthology.orchestra.run.smithi175.stdout:18.2.4-719.gced649f8.el9 2024-08-17T16:57:46.999 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-719.gced649f8.el9 2024-08-17T16:57:46.999 INFO:teuthology.task.install:The correct ceph version 18.2.4-719.gced649f8 is installed. 2024-08-17T16:57:47.000 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-17T16:57:47.001 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:57:47.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:57:47.034 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:57:47.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:57:47.069 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-17T16:57:47.069 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:57:47.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:57:47.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:57:47.178 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:57:47.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:57:47.206 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:57:47.273 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-17T16:57:47.274 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:57:47.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:57:47.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:57:47.371 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:57:47.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:57:47.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:57:47.461 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-17T16:57:47.461 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:57:47.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:57:47.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:57:47.555 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:57:47.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:57:47.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:57:47.645 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-17T16:57:47.742 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:57:47.743 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:57:47.743 INFO:tasks.cephadm:Cluster fsid is d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:57:47.743 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-17T16:57:47.744 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.64', 'mon.c': '[v2:172.21.15.64:3301,v1:172.21.15.64:6790]', 'mon.b': '172.21.15.175'} 2024-08-17T16:57:47.744 INFO:tasks.cephadm:First mon is mon.a on smithi064 2024-08-17T16:57:47.744 INFO:tasks.cephadm:First mgr is y 2024-08-17T16:57:47.744 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-17T16:57:47.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo hostname $(hostname -s) 2024-08-17T16:57:47.772 DEBUG:teuthology.orchestra.run.smithi175:> sudo hostname $(hostname -s) 2024-08-17T16:57:47.798 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-08-17T16:57:47.798 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:57:47.943 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/55847/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-5836-gcd8bdbeb', 'node_name': '172.21.5.36+adami06', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-5836.gcd8bdbeb'}, 'url': 'https://2.chacra.ceph.com/r/ceph/main/cd8bdbeb9fbdab5733aa3aaadc042c965473e453/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-08-15 21:47:48.216668', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://2.chacra.ceph.com/repos/ceph/main/cd8bdbeb9fbdab5733aa3aaadc042c965473e453/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-08-17T16:57:48.202 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref reef, sha1 ced649f89259a8c5733cc18a50f42f20f41ba352 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-08-17T16:57:48.203 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm 2024-08-17T16:57:48.204 INFO:tasks.cephadm:Downloading cephadm from url: https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm 2024-08-17T16:57:48.204 DEBUG:teuthology.orchestra.run.smithi064:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.565 INFO:teuthology.orchestra.run.smithi064.stdout:-rw-r--r--. 1 ubuntu ubuntu 215594 Aug 17 16:57 /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.566 DEBUG:teuthology.orchestra.run.smithi175:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.913 INFO:teuthology.orchestra.run.smithi175.stdout:-rw-r--r--. 1 ubuntu ubuntu 215594 Aug 17 16:57 /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.913 DEBUG:teuthology.orchestra.run.smithi064:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.934 DEBUG:teuthology.orchestra.run.smithi175:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:57:48.961 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-08-17T16:57:48.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-17T16:57:48.977 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-17T16:57:49.799 INFO:teuthology.orchestra.run.smithi175.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:57:49.937 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:58:32.470 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T16:58:32.470 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph_version": "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)", 2024-08-17T16:58:32.470 INFO:teuthology.orchestra.run.smithi175.stdout: "image_id": "c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e", 2024-08-17T16:58:32.470 INFO:teuthology.orchestra.run.smithi175.stdout: "repo_digests": [ 2024-08-17T16:58:32.470 INFO:teuthology.orchestra.run.smithi175.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408" 2024-08-17T16:58:32.471 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T16:58:32.471 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T16:58:32.899 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T16:58:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)", 2024-08-17T16:58:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "image_id": "c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e", 2024-08-17T16:58:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "repo_digests": [ 2024-08-17T16:58:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408" 2024-08-17T16:58:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T16:58:32.900 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T16:58:32.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph 2024-08-17T16:58:32.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo mkdir -p /etc/ceph 2024-08-17T16:58:32.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /etc/ceph 2024-08-17T16:58:33.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 777 /etc/ceph 2024-08-17T16:58:33.061 INFO:tasks.cephadm:Writing seed config... 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-17T16:58:33.063 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-17T16:58:33.064 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:58:33.064 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-17T16:58:33.080 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = d49b6e42-5cb9-11ef-bccf-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-17T16:58:33.080 DEBUG:teuthology.orchestra.run.smithi064:mon.a> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service 2024-08-17T16:58:33.122 DEBUG:teuthology.orchestra.run.smithi064:mgr.y> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service 2024-08-17T16:58:33.165 INFO:tasks.cephadm:Bootstrapping... 2024-08-17T16:58:33.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.64 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:58:33.324 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:58:33.324 INFO:teuthology.orchestra.run.smithi064.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'd49b6e42-5cb9-11ef-bccf-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.64', '--skip-admin-label'] 2024-08-17T16:58:33.351 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:58:33.351 INFO:teuthology.orchestra.run.smithi064.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-08-17T16:58:33.351 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying podman|docker is present... 2024-08-17T16:58:33.376 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:58:33.376 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying lvm2 is present... 2024-08-17T16:58:33.376 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying time synchronization is in place... 2024-08-17T16:58:33.384 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:58:33.384 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:58:33.390 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:58:33.391 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:58:33.397 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout enabled 2024-08-17T16:58:33.404 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout active 2024-08-17T16:58:33.404 INFO:teuthology.orchestra.run.smithi064.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:58:33.405 INFO:teuthology.orchestra.run.smithi064.stdout:Repeating the final host check... 2024-08-17T16:58:33.427 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:58:33.428 INFO:teuthology.orchestra.run.smithi064.stdout:podman (/bin/podman) version 5.1.2 is present 2024-08-17T16:58:33.428 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl is present 2024-08-17T16:58:33.428 INFO:teuthology.orchestra.run.smithi064.stdout:lvcreate is present 2024-08-17T16:58:33.434 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:58:33.435 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:58:33.441 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:58:33.441 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:58:33.448 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout enabled 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout active 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Host looks OK 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Cluster fsid: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Acquiring lock 140308027524864 on /run/cephadm/d49b6e42-5cb9-11ef-bccf-c7b262605968.lock 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Lock 140308027524864 acquired on /run/cephadm/d49b6e42-5cb9-11ef-bccf-c7b262605968.lock 2024-08-17T16:58:33.455 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying IP 172.21.15.64 port 3300 ... 2024-08-17T16:58:33.456 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying IP 172.21.15.64 port 6789 ... 2024-08-17T16:58:33.456 INFO:teuthology.orchestra.run.smithi064.stdout:Base mon IP(s) is [172.21.15.64:3300, 172.21.15.64:6789], mon addrv is [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2024-08-17T16:58:33.459 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.64 metric 100 2024-08-17T16:58:33.459 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.64 metric 100 2024-08-17T16:58:33.462 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-08-17T16:58:33.462 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-08-17T16:58:33.462 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1775sec hoplimit 64 pref medium 2024-08-17T16:58:33.464 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-17T16:58:33.464 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-08-17T16:58:33.465 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:58:33.465 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-08-17T16:58:33.465 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:feb6:f8df/64 scope link 2024-08-17T16:58:33.465 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:58:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2024-08-17T16:58:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2024-08-17T16:58:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-08-17T16:58:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-17T16:58:33.466 INFO:teuthology.orchestra.run.smithi064.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Getting image source signatures 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying blob sha256:090188efbbf262fdcde0da55307c932266150937ffe226cad16cfadb849e480e 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying config sha256:c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e 2024-08-17T16:58:33.935 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Writing manifest to image destination 2024-08-17T16:58:34.848 INFO:teuthology.orchestra.run.smithi064.stdout:ceph: stdout ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable) 2024-08-17T16:58:34.848 INFO:teuthology.orchestra.run.smithi064.stdout:Ceph version: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable) 2024-08-17T16:58:34.848 INFO:teuthology.orchestra.run.smithi064.stdout:Extracting ceph user uid/gid from container image... 2024-08-17T16:58:35.686 INFO:teuthology.orchestra.run.smithi064.stdout:stat: stdout 167 167 2024-08-17T16:58:35.687 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial keys... 2024-08-17T16:58:36.567 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQC81sBmMg81AxAAHhHw7tvkqg3EsVHFXwDjtg== 2024-08-17T16:58:37.506 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQC81sBmqc+0NRAAdkAUxzFRrlH/N0Te2zPDCA== 2024-08-17T16:58:38.378 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQC91sBmiTm0MBAAC5gZnRJhxC/ntqq1ozoZjA== 2024-08-17T16:58:38.379 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial monmap... 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:monmaptool for a [v2:172.21.15.64:3300,v1:172.21.15.64:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:setting min_mon_release = pacific 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: set fsid to d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:58:39.310 INFO:teuthology.orchestra.run.smithi064.stdout:Creating mon... 2024-08-17T16:58:40.258 INFO:teuthology.orchestra.run.smithi064.stdout:create mon.a on 2024-08-17T16:58:40.466 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-08-17T16:58:40.611 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-17T16:58:40.808 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target. 2024-08-17T16:58:40.808 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target. 2024-08-17T16:58:41.021 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a 2024-08-17T16:58:41.021 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to reset failed state of unit ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service: Unit ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service not loaded. 2024-08-17T16:58:41.198 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target.wants/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service → /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@.service. 2024-08-17T16:58:42.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:42 smithi064 podman[30173]: 2024-08-17 16:58:42.632590876 +0000 UTC m=+1.275397663 container create 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T16:58:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:44 smithi064 podman[30173]: 2024-08-17 16:58:44.395437071 +0000 UTC m=+3.038243820 container init 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T16:58:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:44 smithi064 podman[30173]: 2024-08-17 16:58:44.399944146 +0000 UTC m=+3.042750895 container start 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f) 2024-08-17T16:58:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:44 smithi064 ceph-mon[30203]: mkfs d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:44 smithi064 ceph-mon[30203]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:58:45.210 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:58:45.210 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2024-08-17T16:58:45.218 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:58:45.218 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:58:45.218 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2024-08-17T16:58:45.218 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:58:45.219 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mon to start... 2024-08-17T16:58:45.219 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mon... 2024-08-17T16:58:45.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 bash[30173]: 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 2024-08-17T16:58:45.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 systemd[1]: Started Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T16:58:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 ceph-mon[30203]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:58:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 ceph-mon[30203]: monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:58:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 ceph-mon[30203]: fsmap 2024-08-17T16:58:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 ceph-mon[30203]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:58:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:45 smithi064 ceph-mon[30203]: mgrmap e1: no daemons active 2024-08-17T16:58:48.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:48 smithi064 ceph-mon[30203]: from='client.? 172.21.15.64:0/1008408403' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout cluster: 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout id: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout services: 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 3s) 2024-08-17T16:58:52.148 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout data: 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout pgs: 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:mon is available 2024-08-17T16:58:52.149 INFO:teuthology.orchestra.run.smithi064.stdout:Assimilating anything we can from ceph.conf... 2024-08-17T16:58:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:52 smithi064 ceph-mon[30203]: from='client.? 172.21.15.64:0/2902872350' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:58:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:52 smithi064 ceph-mon[30203]: from='client.? 172.21.15.64:0/2902872350' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout fsid = d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:58:53.170 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:58:53.171 INFO:teuthology.orchestra.run.smithi064.stdout:Generating new minimal ceph.conf... 2024-08-17T16:58:53.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:53 smithi064 ceph-mon[30203]: from='client.? 172.21.15.64:0/2618014423' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:54.108 INFO:teuthology.orchestra.run.smithi064.stdout:Restarting the monitor... 2024-08-17T16:58:54.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 systemd[1]: Stopping Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T16:58:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[30199]: 2024-08-17T16:58:54.365+0000 7f6883606640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T16:58:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[30199]: 2024-08-17T16:58:54.365+0000 7f6883606640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-17T16:58:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 podman[30446]: 2024-08-17 16:58:54.386983927 +0000 UTC m=+0.112929857 container died 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T16:58:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 podman[30446]: 2024-08-17 16:58:54.748926226 +0000 UTC m=+0.474872153 container cleanup 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T16:58:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:54 smithi064 bash[30446]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a 2024-08-17T16:58:55.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 podman[30458]: 2024-08-17 16:58:55.033072561 +0000 UTC m=+0.639339936 container remove 0ee635cfa1d2564ad988f17f3f01d51d3d04cb0391b900882e8bcd8bd0353e34 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True) 2024-08-17T16:58:55.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T16:58:55.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 systemd[1]: Stopped Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T16:58:55.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 systemd[1]: Starting Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T16:58:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 podman[30554]: 2024-08-17 16:58:55.302484859 +0000 UTC m=+0.017536977 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:58:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 podman[30554]: 2024-08-17 16:58:55.400362437 +0000 UTC m=+0.115414533 container create b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T16:58:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 podman[30554]: 2024-08-17 16:58:55.636588536 +0000 UTC m=+0.351640641 container init b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T16:58:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 podman[30554]: 2024-08-17 16:58:55.639933426 +0000 UTC m=+0.354985525 container start b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T16:58:55.716 INFO:teuthology.orchestra.run.smithi064.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-08-17T16:58:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:58:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 2 2024-08-17T16:58:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: pidfile_write: ignore empty --pid-file 2024-08-17T16:58:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: load: jerasure load: lrc 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: RocksDB version: 7.9.2 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Git sha 0 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: DB SUMMARY 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: DB Session ID: B6PEWXC0AP3UVHGZN5JO 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84843 ; 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:58:55.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.env: 0x562426921bc0 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.info_log: 0x562428a8abe0 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.statistics: (nil) 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.use_fsync: 0 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:58:55.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.db_log_dir: 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.wal_dir: 2024-08-17T16:58:55.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.write_buffer_manager: 0x562428a5a140 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:58:55.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.unordered_write: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.row_cache: None 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.wal_filter: None 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:58:55.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.wal_compression: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:58:55.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:58:55.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_open_files: -1 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Compression algorithms supported: 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kZSTD supported: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:58:55.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.merge_operator: 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_filter: None 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562428a8a820) 2024-08-17T16:58:55.948 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_type: 0 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_index_type: 0 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_shortening: 1 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: checksum: 4 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: no_block_cache: 0 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache: 0x562428a7f350 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_options: 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: capacity : 536870912 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_shard_bits : 4 2024-08-17T16:58:55.949 INFO:journalctl@ceph.mon.a.smithi064.stdout: strict_capacity_limit : 0 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_compressed: (nil) 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: persistent_cache: (nil) 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size: 4096 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size_deviation: 10 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_restart_interval: 16 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_block_restart_interval: 1 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_block_size: 4096 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: partition_filters: 0 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: use_delta_encoding: 1 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: filter_policy: bloomfilter 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: whole_key_filtering: 1 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: verify_compression: 0 2024-08-17T16:58:55.950 INFO:journalctl@ceph.mon.a.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: format_version: 5 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_index_compression: 1 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_align: 0 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_auto_readahead_size: 262144 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: prepopulate_block_cache: 0 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: initial_auto_readahead_size: 8192 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression: NoCompression 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.num_levels: 7 2024-08-17T16:58:55.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:58:55.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:58:55.953 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:58:55.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.table_properties_collectors: 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:58:55.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.ttl: 2592000 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:58:55.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3bee3356-7a33-4cca-9d5e-d743da59347c 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913935667255, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913935668148, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81219, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 262, "table_properties": {"data_size": 79395, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10297, "raw_average_key_size": 47, "raw_value_size": 73570, "raw_average_value_size": 339, "num_data_blocks": 10, "num_entries": 217, "num_filter_entries": 217, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723913935, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3bee3356-7a33-4cca-9d5e-d743da59347c", "db_session_id": "B6PEWXC0AP3UVHGZN5JO", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913935668234, "job": 1, "event": "recovery_finished"} 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-08-17T16:58:55.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562428aa8e00 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: DB pointer 0x562428b9c000 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** DB Stats ** 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.958 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 2/0 81.15 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 81.15 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.959 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 12.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 12.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache BinnedLRUCache@0x562428a7f350#2 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:58:55.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???) e1 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).mds e1 new map 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).mds e1 print_map 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: e1 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: legacy client fscid: -1 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout: No filesystems configured 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:58:55.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 bash[30554]: b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 systemd[1]: Started Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: fsmap 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:58:55.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:55 smithi064 ceph-mon[30574]: mgrmap e1: no daemons active 2024-08-17T16:58:57.292 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote config to /etc/ceph/ceph.conf 2024-08-17T16:58:57.313 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:58:57.313 INFO:teuthology.orchestra.run.smithi064.stdout:Creating mgr... 2024-08-17T16:58:57.314 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying port 0.0.0.0:9283 ... 2024-08-17T16:58:57.314 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying port 0.0.0.0:8765 ... 2024-08-17T16:58:57.579 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y 2024-08-17T16:58:57.579 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to reset failed state of unit ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service: Unit ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service not loaded. 2024-08-17T16:58:57.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:57 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2548542008' entity='client.admin' 2024-08-17T16:58:57.730 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968.target.wants/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@.service. 2024-08-17T16:58:57.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:57 smithi064 systemd[1]: Starting Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T16:58:58.276 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:57 smithi064 podman[30787]: 2024-08-17 16:58:57.973777203 +0000 UTC m=+0.113718507 container create 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-08-17T16:58:58.276 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:57 smithi064 podman[30787]: 2024-08-17 16:58:57.8772338 +0000 UTC m=+0.017175045 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:58:58.276 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 podman[30787]: 2024-08-17 16:58:58.20935973 +0000 UTC m=+0.349300987 container init 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T16:58:58.276 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 podman[30787]: 2024-08-17 16:58:58.212117373 +0000 UTC m=+0.352058619 container start 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T16:58:58.276 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 bash[30787]: 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 2024-08-17T16:58:58.293 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:58:58.293 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2024-08-17T16:58:58.302 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:58:58.302 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:58:58.302 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2024-08-17T16:58:58.302 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:58:58.312 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:58:58.313 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2024-08-17T16:58:58.321 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:58:58.321 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:58:58.321 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2024-08-17T16:58:58.321 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-08-17T16:58:58.322 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr to start... 2024-08-17T16:58:58.322 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr... 2024-08-17T16:58:58.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 systemd[1]: Started Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T16:58:58.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:58.349+0000 7f9d3dde1040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:58:58.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:58.503+0000 7f9d3dde1040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:58:58.824 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:58.577+0000 7f9d3dde1040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:58:59.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/609690405' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:58:59.125 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:58.838+0000 7f9d3dde1040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:58:59.125 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:58 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:58.901+0000 7f9d3dde1040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:58:59.460 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:58:59.462 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "d49b6e42-5cb9-11ef-bccf-c7b262605968", 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:58:59.463 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:58:59.464 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:58:59.467 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:58:59.468 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:58:59.469 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:58:44.445753+0000", 2024-08-17T16:58:59.470 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:58:59.470 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:58:59.470 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:58:59.470 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:58:59.470 INFO:teuthology.orchestra.run.smithi064.stdout:mgr not available, waiting (1/15)... 2024-08-17T16:59:00.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:58:59 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:58:59.881+0000 7f9d3dde1040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:59:00.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.000+0000 7f9d3dde1040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:00.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.058+0000 7f9d3dde1040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:59:00.558 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.276+0000 7f9d3dde1040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:59:00.558 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.339+0000 7f9d3dde1040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:59:00.558 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.407+0000 7f9d3dde1040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:59:00.558 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.556+0000 7f9d3dde1040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:59:01.316 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:00 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:00.991+0000 7f9d3dde1040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:59:01.316 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:01.314+0000 7f9d3dde1040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:59:01.593 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:01.384+0000 7f9d3dde1040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:59:01.594 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:01.548+0000 7f9d3dde1040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:59:02.134 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:01.879+0000 7f9d3dde1040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:59:02.134 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:01.945+0000 7f9d3dde1040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:59:02.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:02.142+0000 7f9d3dde1040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:59:02.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:02.304+0000 7f9d3dde1040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:59:02.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:02.371+0000 7f9d3dde1040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:59:02.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/192283366' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:59:02.848 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:02.461+0000 7f9d3dde1040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:02.848 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:02.525+0000 7f9d3dde1040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:59:03.148 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:03.082+0000 7f9d3dde1040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "d49b6e42-5cb9-11ef-bccf-c7b262605968", 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:59:03.153 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:59:03.154 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:59:03.155 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:59:03.158 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:59:03.158 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:59:03.158 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.158 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:59:03.158 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:59:03.159 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:58:44.445753+0000", 2024-08-17T16:59:03.160 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:59:03.161 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:03.161 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:59:03.161 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:03.161 INFO:teuthology.orchestra.run.smithi064.stdout:mgr not available, waiting (2/15)... 2024-08-17T16:59:03.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:03.247+0000 7f9d3dde1040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:59:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: Activating manager daemon y 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: mgrmap e2: y(active, starting, since 0.00315454s) 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: Manager daemon y is now available 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' 2024-08-17T16:59:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:03 smithi064 ceph-mon[30574]: from='mgr.14100 172.21.15.64:0/1382032504' entity='mgr.y' 2024-08-17T16:59:05.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:05 smithi064 ceph-mon[30574]: mgrmap e3: y(active, since 1.00729s) 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "d49b6e42-5cb9-11ef-bccf-c7b262605968", 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:59:06.426 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:59:06.427 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:59:06.428 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:59:06.431 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:59:06.431 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:59:06.431 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:06.432 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:59:06.433 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:58:44.445753+0000", 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:06.434 INFO:teuthology.orchestra.run.smithi064.stdout:mgr is available 2024-08-17T16:59:06.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:06 smithi064 ceph-mon[30574]: mgrmap e4: y(active, since 2s) 2024-08-17T16:59:06.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:06 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1905823283' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:59:07.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:07 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/565816831' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:59:07.522 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:59:07.522 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:59:07.522 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout fsid = d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:59:07.522 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:59:07.522 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:59:07.523 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling cephadm module... 2024-08-17T16:59:08.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/659518744' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-17T16:59:08.556 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setuser ceph since I am not root 2024-08-17T16:59:08.556 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setgroup ceph since I am not root 2024-08-17T16:59:08.556 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:08.404+0000 7f5adb5ee040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:59:08.556 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:08.554+0000 7f5adb5ee040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:59:08.858 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:08.624+0000 7f5adb5ee040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:59:09.186 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:08.884+0000 7f5adb5ee040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:59:09.186 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:08.950+0000 7f5adb5ee040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:59:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:09 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/659518744' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-17T16:59:09.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:09 smithi064 ceph-mon[30574]: mgrmap e5: y(active, since 5s) 2024-08-17T16:59:10.104 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:09.925+0000 7f5adb5ee040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:59:10.104 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.044+0000 7f5adb5ee040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:10.105 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.103+0000 7f5adb5ee040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:59:10.134 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:10.134 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-08-17T16:59:10.134 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:59:10.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:59:10.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:59:10.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:10.135 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for the mgr to restart... 2024-08-17T16:59:10.135 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr epoch 5... 2024-08-17T16:59:10.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2847308582' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:59:10.359 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.325+0000 7f5adb5ee040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:59:10.613 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.391+0000 7f5adb5ee040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:59:10.613 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.459+0000 7f5adb5ee040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:59:10.613 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:10.611+0000 7f5adb5ee040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:59:11.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.046+0000 7f5adb5ee040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:59:11.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.370+0000 7f5adb5ee040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:59:11.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.438+0000 7f5adb5ee040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:59:11.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.589+0000 7f5adb5ee040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:59:12.166 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.909+0000 7f5adb5ee040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:59:12.167 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:11.972+0000 7f5adb5ee040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:59:12.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:12.165+0000 7f5adb5ee040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:59:12.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:12.319+0000 7f5adb5ee040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:59:12.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:12.385+0000 7f5adb5ee040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:59:12.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:12.470+0000 7f5adb5ee040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:12.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:12.531+0000 7f5adb5ee040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:59:13.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:13.080+0000 7f5adb5ee040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:59:13.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:13.242+0000 7f5adb5ee040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: Active manager daemon y restarted 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: Activating manager daemon y 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: osdmap e2: 0 total, 0 up, 0 in 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: mgrmap e6: y(active, starting, since 0.00324222s) 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: Manager daemon y is now available 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:13 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:14 smithi064 ceph-mon[30574]: Found migration_current of "None". Setting to last migration. 2024-08-17T16:59:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:14 smithi064 ceph-mon[30574]: mgrmap e7: y(active, since 1.00487s) 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:mgr epoch 5 is available 2024-08-17T16:59:14.849 INFO:teuthology.orchestra.run.smithi064.stdout:Setting orchestrator backend to cephadm... 2024-08-17T16:59:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:15 smithi064 ceph-mon[30574]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T16:59:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:15 smithi064 ceph-mon[30574]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T16:59:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:15 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:15 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:16.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-mon[30574]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:16.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:16.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:16.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-mon[30574]: mgrmap e8: y(active, since 2s) 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:16.549+0000 7f5a4347d640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:59:16] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Traceback (most recent call last): 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._connections.run(self.expiration_interval) 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._run(expiration_interval) 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:16.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s = self.context.wrap_socket( 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: return self.sslsocket_class._create( 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self.do_handshake() 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._sslobj.do_handshake() 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:16.656 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:17.194 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout value unchanged 2024-08-17T16:59:17.194 INFO:teuthology.orchestra.run.smithi064.stdout:Generating ssh key... 2024-08-17T16:59:17.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:16] ENGINE Bus STARTING 2024-08-17T16:59:17.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:16] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:16] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:16] ENGINE Bus STARTED 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:16] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:17.620 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T16:59:17.621 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:17.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:17 smithi064 ceph-mon[30574]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:18.431 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Generating public/private rsa key pair. 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Your identification has been saved in /tmp/tmpogjdj1ge/key 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Your public key has been saved in /tmp/tmpogjdj1ge/key.pub 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: The key fingerprint is: 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: SHA256:Ckjz6P2n/Ft+eBRX6o5yzMPbBfg/7TxYbgkdOCwVZ78 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: The key's randomart image is: 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: +---[RSA 3072]----+ 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | ..o | 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | .o o| 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | o o .o.| 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | . = ..=o..| 2024-08-17T16:59:18.432 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | o o S o++E.| 2024-08-17T16:59:18.433 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | . . . . .o.+ | 2024-08-17T16:59:18.433 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | . . . .* o* +| 2024-08-17T16:59:18.433 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | o .oo Xo.X.| 2024-08-17T16:59:18.433 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: | ++o..=.oo.*| 2024-08-17T16:59:18.433 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: +----[SHA256]-----+ 2024-08-17T16:59:19.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:19 smithi064 ceph-mon[30574]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:19.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:19 smithi064 ceph-mon[30574]: Generating ssh key... 2024-08-17T16:59:19.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:19 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:19.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:19 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:19.873 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCo+XCO803U03Vi/7aB6wky3zwz+D8yj0VunEkXgCB6BVZMraqdOUTuvn2iIwyszJwh5jaB85Lo89ZldUux8fyRRTiQH/WtuG6hUXsFYWYNvyFhd2EEcOHnTGBYLosgi74GDytiB4KkiI89gYnlvXT2yXale0sptahwmBYTdNuIlGnA6N7b7L8Mf7Jd1DQ8CkWWP6IOoOS1unzTso1uxJK3CZg2+ShFmiRVSbGJWi2nhsYPghYd8HbChMVABpxm9s3ApBfPw0Tl66ygRWsaoAG/ShmD16NPTBO2yMbduytFlgQXFHMOx36Nk8NO6X5xwN87gcldKs4tNSy7lN0HKHE8BpPK8q65BX7zhXn1pPKrNvFqvMLbBOYabzEj1grwG/bLpxBGX8TOroCiXvK+1X/aZa3n7yE3DbddaeFvtdOjyI+INGUo/QFLWNnIBfhb3qPF+UIr8QT9Sz9sJd4CpceLDd80UHIDRK+q2bvy/l8hhDXe0lRASz33R/8mDDuIfXs= ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:59:19.873 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-17T16:59:19.874 INFO:teuthology.orchestra.run.smithi064.stdout:Adding key to root@localhost authorized_keys... 2024-08-17T16:59:19.874 INFO:teuthology.orchestra.run.smithi064.stdout:Adding host smithi064... 2024-08-17T16:59:20.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:20 smithi064 ceph-mon[30574]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:21.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:21 smithi064 ceph-mon[30574]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi064", "addr": "172.21.15.64", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:22 smithi064 ceph-mon[30574]: Deploying cephadm binary to smithi064 2024-08-17T16:59:24.356 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Added host 'smithi064' with addr '172.21.15.64' 2024-08-17T16:59:24.356 INFO:teuthology.orchestra.run.smithi064.stdout:Deploying unmanaged mon service... 2024-08-17T16:59:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:24 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:24 smithi064 ceph-mon[30574]: Added host smithi064 2024-08-17T16:59:24.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:24 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:25.605 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-08-17T16:59:25.606 INFO:teuthology.orchestra.run.smithi064.stdout:Deploying unmanaged mgr service... 2024-08-17T16:59:26.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:25 smithi064 ceph-mon[30574]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:26.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:25 smithi064 ceph-mon[30574]: Saving service mon spec with placement count:5 2024-08-17T16:59:26.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:25 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:27.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:27 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:27.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:27 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:27.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:27 smithi064 ceph-mon[30574]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:27 smithi064 ceph-mon[30574]: Saving service mgr spec with placement count:2 2024-08-17T16:59:27.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:27 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:28.269 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-08-17T16:59:28.905 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:28 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:28.905 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:28 smithi064 ceph-mon[30574]: from='mgr.14118 172.21.15.64:0/2073132658' entity='mgr.y' 2024-08-17T16:59:30.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:29 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3323834345' entity='client.admin' 2024-08-17T16:59:31.354 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling the dashboard module... 2024-08-17T16:59:31.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:31 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1369224777' entity='client.admin' 2024-08-17T16:59:32.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/835044489' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-17T16:59:32.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setuser ceph since I am not root 2024-08-17T16:59:32.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setgroup ceph since I am not root 2024-08-17T16:59:32.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:32.383+0000 7f9b04103040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:59:32.874 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:32.542+0000 7f9b04103040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:59:32.875 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:32.614+0000 7f9b04103040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:59:32.875 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:32.873+0000 7f9b04103040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:59:33.152 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:32.940+0000 7f9b04103040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:59:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:33 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/835044489' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-17T16:59:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:33 smithi064 ceph-mon[30574]: mgrmap e9: y(active, since 18s) 2024-08-17T16:59:34.132 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:33.923+0000 7f9b04103040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:59:34.132 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.043+0000 7f9b04103040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:34.132 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.102+0000 7f9b04103040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:34.135 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for the mgr to restart... 2024-08-17T16:59:34.136 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr epoch 9... 2024-08-17T16:59:34.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2996004384' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:59:34.388 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.335+0000 7f9b04103040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:59:34.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.401+0000 7f9b04103040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:59:34.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.470+0000 7f9b04103040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:59:34.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:34.621+0000 7f9b04103040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:59:35.385 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.059+0000 7f9b04103040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:59:35.386 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.384+0000 7f9b04103040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:59:35.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.452+0000 7f9b04103040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:59:35.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.605+0000 7f9b04103040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:59:36.181 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.924+0000 7f9b04103040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:59:36.181 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:35 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:35.988+0000 7f9b04103040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:59:36.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:36 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:36.179+0000 7f9b04103040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:59:36.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:36 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:36.334+0000 7f9b04103040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:59:36.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:36 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:36.401+0000 7f9b04103040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:59:36.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:36 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:36.487+0000 7f9b04103040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:59:36.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:36 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:36.548+0000 7f9b04103040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:59:37.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:37.097+0000 7f9b04103040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:37.260+0000 7f9b04103040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: Active manager daemon y restarted 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: Activating manager daemon y 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: osdmap e3: 0 total, 0 up, 0 in 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: mgrmap e10: y(active, starting, since 0.00320862s) 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:37.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: Manager daemon y is now available 2024-08-17T16:59:37.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:37.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:37.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:39.035 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:59:39.035 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-08-17T16:59:39.035 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:59:39.035 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:59:39.035 INFO:teuthology.orchestra.run.smithi064.stdout:mgr epoch 9 is available 2024-08-17T16:59:39.036 INFO:teuthology.orchestra.run.smithi064.stdout:Generating a dashboard self-signed certificate... 2024-08-17T16:59:39.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-mon[30574]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T16:59:39.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-mon[30574]: mgrmap e11: y(active, since 1.00569s) 2024-08-17T16:59:39.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-mon[30574]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T16:59:39.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:39.070+0000 7f9a6a741640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:59:39] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Traceback (most recent call last): 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._connections.run(self.expiration_interval) 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._run(expiration_interval) 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:39.371 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s = self.context.wrap_socket( 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: return self.sslsocket_class._create( 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self.do_handshake() 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._sslobj.do_handshake() 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 16:59:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T16:59:40.233 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-08-17T16:59:40.233 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial admin user... 2024-08-17T16:59:40.462 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:38] ENGINE Bus STARTING 2024-08-17T16:59:40.462 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:38] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T16:59:40.462 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:39] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:39] ENGINE Bus STARTED 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: [17/Aug/2024:16:59:39] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:40.463 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:40.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:40 smithi064 ceph-mon[30574]: mgrmap e12: y(active, since 2s) 2024-08-17T16:59:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:41 smithi064 ceph-mon[30574]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:42.161 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$ALu.YCzKyRuJy3T4RTUsi.x/42VuVF8piOKyIQawR.em9xaLQNlby", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1723913981, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-17T16:59:42.162 INFO:teuthology.orchestra.run.smithi064.stdout:Fetching dashboard port number... 2024-08-17T16:59:42.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:42 smithi064 ceph-mon[30574]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:43.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 8443 2024-08-17T16:59:43.378 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:59:43.378 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2024-08-17T16:59:43.386 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:59:43.386 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2024-08-17T16:59:43.386 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2024-08-17T16:59:43.386 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout:Ceph Dashboard is now available at: 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout: URL: https://smithi064.front.sepia.ceph.com:8443/ 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout: User: admin 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout: Password: 3p08h51r72 2024-08-17T16:59:43.388 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:43.389 INFO:teuthology.orchestra.run.smithi064.stdout:Saving cluster configuration to /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config directory 2024-08-17T16:59:43.389 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling autotune for osd_memory_target 2024-08-17T16:59:43.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:43 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1814289737' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-17T16:59:44.524 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:44 smithi064 ceph-mon[30574]: mgrmap e13: y(active, since 6s) 2024-08-17T16:59:45.707 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:Or, if you are only running a single cluster on this host: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:Please consider enabling telemetry to help improve Ceph: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: ceph telemetry on 2024-08-17T16:59:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.709 INFO:teuthology.orchestra.run.smithi064.stdout:For more information see: 2024-08-17T16:59:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-08-17T16:59:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:45.709 INFO:teuthology.orchestra.run.smithi064.stdout:Bootstrap complete. 2024-08-17T16:59:45.741 INFO:tasks.cephadm:Fetching config... 2024-08-17T16:59:45.741 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:59:45.741 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-17T16:59:45.758 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-17T16:59:45.759 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:59:45.759 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-17T16:59:45.815 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-17T16:59:45.816 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:59:45.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-17T16:59:45.881 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-17T16:59:45.881 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T16:59:45.881 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-17T16:59:45.938 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-17T16:59:45.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCo+XCO803U03Vi/7aB6wky3zwz+D8yj0VunEkXgCB6BVZMraqdOUTuvn2iIwyszJwh5jaB85Lo89ZldUux8fyRRTiQH/WtuG6hUXsFYWYNvyFhd2EEcOHnTGBYLosgi74GDytiB4KkiI89gYnlvXT2yXale0sptahwmBYTdNuIlGnA6N7b7L8Mf7Jd1DQ8CkWWP6IOoOS1unzTso1uxJK3CZg2+ShFmiRVSbGJWi2nhsYPghYd8HbChMVABpxm9s3ApBfPw0Tl66ygRWsaoAG/ShmD16NPTBO2yMbduytFlgQXFHMOx36Nk8NO6X5xwN87gcldKs4tNSy7lN0HKHE8BpPK8q65BX7zhXn1pPKrNvFqvMLbBOYabzEj1grwG/bLpxBGX8TOroCiXvK+1X/aZa3n7yE3DbddaeFvtdOjyI+INGUo/QFLWNnIBfhb3qPF+UIr8QT9Sz9sJd4CpceLDd80UHIDRK+q2bvy/l8hhDXe0lRASz33R/8mDDuIfXs= ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:59:46.013 INFO:teuthology.orchestra.run.smithi064.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCo+XCO803U03Vi/7aB6wky3zwz+D8yj0VunEkXgCB6BVZMraqdOUTuvn2iIwyszJwh5jaB85Lo89ZldUux8fyRRTiQH/WtuG6hUXsFYWYNvyFhd2EEcOHnTGBYLosgi74GDytiB4KkiI89gYnlvXT2yXale0sptahwmBYTdNuIlGnA6N7b7L8Mf7Jd1DQ8CkWWP6IOoOS1unzTso1uxJK3CZg2+ShFmiRVSbGJWi2nhsYPghYd8HbChMVABpxm9s3ApBfPw0Tl66ygRWsaoAG/ShmD16NPTBO2yMbduytFlgQXFHMOx36Nk8NO6X5xwN87gcldKs4tNSy7lN0HKHE8BpPK8q65BX7zhXn1pPKrNvFqvMLbBOYabzEj1grwG/bLpxBGX8TOroCiXvK+1X/aZa3n7yE3DbddaeFvtdOjyI+INGUo/QFLWNnIBfhb3qPF+UIr8QT9Sz9sJd4CpceLDd80UHIDRK+q2bvy/l8hhDXe0lRASz33R/8mDDuIfXs= ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:59:46.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCo+XCO803U03Vi/7aB6wky3zwz+D8yj0VunEkXgCB6BVZMraqdOUTuvn2iIwyszJwh5jaB85Lo89ZldUux8fyRRTiQH/WtuG6hUXsFYWYNvyFhd2EEcOHnTGBYLosgi74GDytiB4KkiI89gYnlvXT2yXale0sptahwmBYTdNuIlGnA6N7b7L8Mf7Jd1DQ8CkWWP6IOoOS1unzTso1uxJK3CZg2+ShFmiRVSbGJWi2nhsYPghYd8HbChMVABpxm9s3ApBfPw0Tl66ygRWsaoAG/ShmD16NPTBO2yMbduytFlgQXFHMOx36Nk8NO6X5xwN87gcldKs4tNSy7lN0HKHE8BpPK8q65BX7zhXn1pPKrNvFqvMLbBOYabzEj1grwG/bLpxBGX8TOroCiXvK+1X/aZa3n7yE3DbddaeFvtdOjyI+INGUo/QFLWNnIBfhb3qPF+UIr8QT9Sz9sJd4CpceLDd80UHIDRK+q2bvy/l8hhDXe0lRASz33R/8mDDuIfXs= ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:59:46.065 INFO:teuthology.orchestra.run.smithi175.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCo+XCO803U03Vi/7aB6wky3zwz+D8yj0VunEkXgCB6BVZMraqdOUTuvn2iIwyszJwh5jaB85Lo89ZldUux8fyRRTiQH/WtuG6hUXsFYWYNvyFhd2EEcOHnTGBYLosgi74GDytiB4KkiI89gYnlvXT2yXale0sptahwmBYTdNuIlGnA6N7b7L8Mf7Jd1DQ8CkWWP6IOoOS1unzTso1uxJK3CZg2+ShFmiRVSbGJWi2nhsYPghYd8HbChMVABpxm9s3ApBfPw0Tl66ygRWsaoAG/ShmD16NPTBO2yMbduytFlgQXFHMOx36Nk8NO6X5xwN87gcldKs4tNSy7lN0HKHE8BpPK8q65BX7zhXn1pPKrNvFqvMLbBOYabzEj1grwG/bLpxBGX8TOroCiXvK+1X/aZa3n7yE3DbddaeFvtdOjyI+INGUo/QFLWNnIBfhb3qPF+UIr8QT9Sz9sJd4CpceLDd80UHIDRK+q2bvy/l8hhDXe0lRASz33R/8mDDuIfXs= ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T16:59:46.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-17T16:59:46.275 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:46 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/777185742' entity='client.admin' 2024-08-17T16:59:46.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:49.317 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-17T16:59:49.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-17T16:59:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:49 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1529542602' entity='client.admin' 2024-08-17T16:59:50.074 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:51.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:51.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:59:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:52.387 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi175 2024-08-17T16:59:52.387 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:59:52.387 DEBUG:teuthology.orchestra.run.smithi175:> dd of=/etc/ceph/ceph.conf 2024-08-17T16:59:52.407 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T16:59:52.408 DEBUG:teuthology.orchestra.run.smithi175:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:52.465 INFO:tasks.cephadm:Adding host smithi175 to orchestrator... 2024-08-17T16:59:52.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch host add smithi175 2024-08-17T16:59:52.674 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:53.801 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:53 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:54 smithi064 ceph-mon[30574]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:56 smithi064 ceph-mon[30574]: Deploying cephadm binary to smithi175 2024-08-17T16:59:57.510 INFO:teuthology.orchestra.run.smithi064.stdout:Added host 'smithi175' with addr '172.21.15.175' 2024-08-17T16:59:58.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch host ls --format=json 2024-08-17T16:59:58.295 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:58.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:58 smithi064 ceph-mon[30574]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:59:58.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:58.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:58.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:58.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:59.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:59 smithi064 ceph-mon[30574]: Added host smithi175 2024-08-17T16:59:59.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 16:59:59 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T16:59:59.994 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T16:59:59.994 INFO:teuthology.orchestra.run.smithi064.stdout:[{"addr": "172.21.15.64", "hostname": "smithi064", "labels": [], "status": ""}, {"addr": "172.21.15.175", "hostname": "smithi175", "labels": [], "status": ""}] 2024-08-17T17:00:00.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:00 smithi064 ceph-mon[30574]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:00.595 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-17T17:00:00.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd crush tunables default 2024-08-17T17:00:00.800 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:01 smithi064 ceph-mon[30574]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:00:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi175", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:03.296 INFO:teuthology.orchestra.run.smithi064.stderr:adjusted tunables profile to default 2024-08-17T17:00:03.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:03.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:03.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:03.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:03.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:03.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:03.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:03.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:03 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T17:00:03.967 INFO:tasks.cephadm:Adding mon.a on smithi064 2024-08-17T17:00:03.967 INFO:tasks.cephadm:Adding mon.c on smithi064 2024-08-17T17:00:03.967 INFO:tasks.cephadm:Adding mon.b on smithi175 2024-08-17T17:00:03.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply mon '3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b' 2024-08-17T17:00:04.171 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:04.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:04 smithi064 ceph-mon[30574]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:04.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:04 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:00:04.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:04 smithi064 ceph-mon[30574]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:05.702 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled mon update... 2024-08-17T17:00:06.295 DEBUG:teuthology.orchestra.run.smithi064:mon.c> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.c.service 2024-08-17T17:00:06.298 DEBUG:teuthology.orchestra.run.smithi175:mon.b> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service 2024-08-17T17:00:06.301 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:00:06.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:00:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:06.915 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:07 smithi064 ceph-mon[30574]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:07 smithi064 ceph-mon[30574]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b;count:3 2024-08-17T17:00:07.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:07 smithi064 ceph-mon[30574]: Deploying daemon mon.b on smithi175 2024-08-17T17:00:08.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:08 smithi064 ceph-mon[30574]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:09 smithi064 ceph-mon[30574]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:10.116 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:00:10.116 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","modified":"2024-08-17T16:58:38.717878Z","created":"2024-08-17T16:58:38.717878Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-17T17:00:10.116 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-08-17T17:00:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:10 smithi175 systemd[1]: Starting Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:00:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:10 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/3551848139' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:10 smithi175 podman[34586]: 2024-08-17 17:00:10.74908775 +0000 UTC m=+0.017181827 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:10 smithi175 podman[34586]: 2024-08-17 17:00:10.847577364 +0000 UTC m=+0.115671435 container create 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 podman[34586]: 2024-08-17 17:00:11.025690367 +0000 UTC m=+0.293784440 container init 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 podman[34586]: 2024-08-17 17:00:11.02907651 +0000 UTC m=+0.297170578 container start 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 2 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pidfile_write: ignore empty --pid-file 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: load: jerasure load: lrc 2024-08-17T17:00:11.101 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Git sha 0 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: DB SUMMARY 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: DB Session ID: 6NLOFRGUU9QFS1H0Z0AO 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-08-17T17:00:11.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:00:11.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:00:11.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:00:11.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:00:11.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:00:11.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.env: 0x55f6b4e14bc0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.info_log: 0x55f6b588cbe0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.statistics: (nil) 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.use_fsync: 0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:00:11.104 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.db_log_dir: 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.wal_dir: 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:00:11.105 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.write_buffer_manager: 0x55f6b585c140 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.unordered_write: 0 2024-08-17T17:00:11.106 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.row_cache: None 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.wal_filter: None 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.wal_compression: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:00:11.107 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:00:11.108 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_open_files: -1 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Compression algorithms supported: 2024-08-17T17:00:11.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kZSTD supported: 0 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:00:11.110 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.merge_operator: 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_filter: None 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f6b588c820) 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_type: 0 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: data_block_index_type: 0 2024-08-17T17:00:11.111 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_shortening: 1 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: checksum: 4 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: no_block_cache: 0 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache: 0x55f6b5881350 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_options: 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: capacity : 536870912 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: num_shard_bits : 4 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: strict_capacity_limit : 0 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_compressed: (nil) 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: persistent_cache: (nil) 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_size: 4096 2024-08-17T17:00:11.112 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_size_deviation: 10 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_restart_interval: 16 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_block_restart_interval: 1 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: metadata_block_size: 4096 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: partition_filters: 0 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: use_delta_encoding: 1 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: filter_policy: bloomfilter 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: whole_key_filtering: 1 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: verify_compression: 0 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: format_version: 5 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: enable_index_compression: 1 2024-08-17T17:00:11.113 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_align: 0 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout: max_auto_readahead_size: 262144 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout: prepopulate_block_cache: 0 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression: NoCompression 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.num_levels: 7 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:00:11.114 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:00:11.115 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:11.116 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:00:11.117 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:00:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.table_properties_collectors: 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:00:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.ttl: 2592000 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:00:11.120 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: afd1e43a-e081-4717-9fc9-1ba4119c5ecd 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914011056762, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T17:00:11.121 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914011057319, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914011, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "afd1e43a-e081-4717-9fc9-1ba4119c5ecd", "db_session_id": "6NLOFRGUU9QFS1H0Z0AO", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914011057445, "job": 1, "event": "recovery_finished"} 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f6b58aae00 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: DB pointer 0x55f6b599e000 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** DB Stats ** 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:11.122 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:11.123 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:00:11.124 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: Block cache BinnedLRUCache@0x55f6b5881350#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: using public_addr v2:172.21.15.175:0/0 -> [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: starting mon.b rank -1 at public addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] at bind addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(???) e0 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).mds e1 new map 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).mds e1 print_map 2024-08-17T17:00:11.125 INFO:journalctl@ceph.mon.b.smithi175.stdout: e1 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout: legacy client fscid: -1 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout: No filesystems configured 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T17:00:11.126 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi175", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:11.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:11.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b;count:3 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: Deploying daemon mon.b on smithi175 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/3551848139' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 ceph-mon[34606]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 bash[34586]: 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 2024-08-17T17:00:11.129 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:11 smithi175 systemd[1]: Started Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:00:11.678 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:00:11.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:00:11.877 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:00:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: mon.a calling monitor election 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: mon.b calling monitor election 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: fsmap 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: mgrmap e13: y(active, since 38s) 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: overall HEALTH_OK 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: mon.a calling monitor election 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: mon.b calling monitor election 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: fsmap 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: mgrmap e13: y(active, since 38s) 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: overall HEALTH_OK 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:16.522 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:00:16.522 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":2,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","modified":"2024-08-17T17:00:11.086692Z","created":"2024-08-17T16:58:38.717878Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-17T17:00:16.522 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 2 2024-08-17T17:00:17.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:17 smithi064 ceph-mon[30574]: Deploying daemon mon.c on smithi064 2024-08-17T17:00:17.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:17 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/608710743' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:17.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:17 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:17.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:17 smithi175 ceph-mon[34606]: Deploying daemon mon.c on smithi064 2024-08-17T17:00:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:17 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/608710743' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:17 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:18.077 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:00:18.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:00:18.290 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:00:18.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:18 smithi064 ceph-mon[30574]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:18.389 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:18 smithi175 ceph-mon[34606]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:18.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:18 smithi064 systemd[1]: Starting Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:00:19.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 podman[40937]: 2024-08-17 17:00:18.947954304 +0000 UTC m=+0.018759663 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:00:19.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 podman[40937]: 2024-08-17 17:00:19.049955202 +0000 UTC m=+0.120760552 container create fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:00:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[30574]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 podman[40937]: 2024-08-17 17:00:19.285673531 +0000 UTC m=+0.356478882 container init fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 podman[40937]: 2024-08-17 17:00:19.289737568 +0000 UTC m=+0.360542919 container start fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 2 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pidfile_write: ignore empty --pid-file 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: load: jerasure load: lrc 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:00:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Git sha 0 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: DB SUMMARY 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: DB Session ID: XVG4D89TVRWAQNQWKEM8 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:00:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.env: 0x555763458bc0 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.info_log: 0x555765256be0 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.statistics: (nil) 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.use_fsync: 0 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:00:19.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.db_log_dir: 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.wal_dir: 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:00:19.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.write_buffer_manager: 0x555765226140 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:00:19.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.unordered_write: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.row_cache: None 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.wal_filter: None 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.wal_compression: 0 2024-08-17T17:00:19.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:00:19.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_open_files: -1 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Compression algorithms supported: 2024-08-17T17:00:19.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kZSTD supported: 0 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.merge_operator: 2024-08-17T17:00:19.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_filter: None 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555765256820) 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_type: 0 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_index_type: 0 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_shortening: 1 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:00:19.447 INFO:journalctl@ceph.mon.c.smithi064.stdout: checksum: 4 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: no_block_cache: 0 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache: 0x55576524b350 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_options: 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: capacity : 536870912 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_shard_bits : 4 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: strict_capacity_limit : 0 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_compressed: (nil) 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: persistent_cache: (nil) 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size: 4096 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size_deviation: 10 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_restart_interval: 16 2024-08-17T17:00:19.448 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_block_restart_interval: 1 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_block_size: 4096 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: partition_filters: 0 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: use_delta_encoding: 1 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: filter_policy: bloomfilter 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: whole_key_filtering: 1 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: verify_compression: 0 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: format_version: 5 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_index_compression: 1 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_align: 0 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_auto_readahead_size: 262144 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: prepopulate_block_cache: 0 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:00:19.449 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression: NoCompression 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.num_levels: 7 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:00:19.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:00:19.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:00:19.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:00:19.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.table_properties_collectors: 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:00:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.ttl: 2592000 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:00:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 609b92da-c0bc-488a-b55a-26ffc658d8d9 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914019317587, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914019317994, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914019, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "609b92da-c0bc-488a-b55a-26ffc658d8d9", "db_session_id": "XVG4D89TVRWAQNQWKEM8", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914019318095, "job": 1, "event": "recovery_finished"} 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-08-17T17:00:19.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555765274e00 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: DB pointer 0x555765368000 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** DB Stats ** 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.458 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.459 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache BinnedLRUCache@0x55576524b350#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: using public_addrv [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: starting mon.c rank -1 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:19.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(???) e0 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).mds e1 new map 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).mds e1 print_map 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: e1 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: legacy client fscid: -1 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout: No filesystems configured 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T17:00:19.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi175", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/330196448' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:19.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi175:172.21.15.175=b;count:3 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Deploying daemon mon.b on smithi175 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/3551848139' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.a calling monitor election 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.b calling monitor election 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: fsmap 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mgrmap e13: y(active, since 38s) 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: overall HEALTH_OK 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: Deploying daemon mon.c on smithi064 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/608710743' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:19.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:19.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:19.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 ceph-mon[40964]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T17:00:19.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 bash[40937]: fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a 2024-08-17T17:00:19.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:19 smithi064 systemd[1]: Started Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:00:19.542 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:19 smithi175 ceph-mon[34606]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: mon.a calling monitor election 2024-08-17T17:00:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: mon.b calling monitor election 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: mon.c calling monitor election 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: fsmap 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: mgrmap e13: y(active, since 47s) 2024-08-17T17:00:24.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: overall HEALTH_OK 2024-08-17T17:00:24.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: mon.a calling monitor election 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: mon.b calling monitor election 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: mon.c calling monitor election 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: fsmap 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: mgrmap e13: y(active, since 47s) 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: overall HEALTH_OK 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: mon.a calling monitor election 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:24.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: mon.b calling monitor election 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: mon.c calling monitor election 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: fsmap 2024-08-17T17:00:24.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: mgrmap e13: y(active, since 47s) 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: overall HEALTH_OK 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:24.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[30574]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[40964]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:25 smithi175 ceph-mon[34606]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:25.949 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:00:25.949 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":3,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","modified":"2024-08-17T17:00:19.343721Z","created":"2024-08-17T16:58:38.717878Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3301","nonce":0},{"type":"v1","addr":"172.21.15.64:6790","nonce":0}]},"addr":"172.21.15.64:6790/0","public_addr":"172.21.15.64:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-08-17T17:00:25.949 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 3 2024-08-17T17:00:26.504 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.504 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/1966745111' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.505 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:26 smithi175 ceph-mon[34606]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:00:26.525 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-17T17:00:26.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph config generate-minimal-conf 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/1966745111' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[40964]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/1966745111' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:26.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:26 smithi064 ceph-mon[30574]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:00:27.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:27.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:27.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:27.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:27.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:27.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:27.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:28.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:28 smithi064 ceph-mon[30574]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:00:28.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:28 smithi064 ceph-mon[30574]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:00:28.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:28 smithi064 ceph-mon[40964]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:00:28.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:28 smithi064 ceph-mon[40964]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:00:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:28 smithi175 ceph-mon[34606]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:00:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:28 smithi175 ceph-mon[34606]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:00:29.577 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:29 smithi064 ceph-mon[30574]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:29.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:29 smithi064 ceph-mon[40964]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:29.600 INFO:teuthology.orchestra.run.smithi064.stdout:# minimal ceph.conf for d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:29.600 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2024-08-17T17:00:29.600 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:00:29.601 INFO:teuthology.orchestra.run.smithi064.stdout: mon_host = [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2024-08-17T17:00:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:29 smithi175 ceph-mon[34606]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:30.473 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-17T17:00:30.474 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:00:30.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T17:00:30.506 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:00:30.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:30.575 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T17:00:30.575 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T17:00:30.602 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T17:00:30.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:30.667 INFO:tasks.cephadm:Adding mgr.y on smithi064 2024-08-17T17:00:30.667 INFO:tasks.cephadm:Adding mgr.x on smithi175 2024-08-17T17:00:30.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply mgr '2;smithi064=y;smithi175=x' 2024-08-17T17:00:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2968508476' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[30574]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2968508476' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:30.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:30 smithi064 ceph-mon[40964]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2968508476' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:30 smithi175 ceph-mon[34606]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:00:31.233 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:00:32.406 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.406 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.406 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:32.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:32.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:32.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:32 smithi175 ceph-mon[34606]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[30574]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:32 smithi064 ceph-mon[40964]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:32.902 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled mgr update... 2024-08-17T17:00:33.486 DEBUG:teuthology.orchestra.run.smithi175:mgr.x> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service 2024-08-17T17:00:33.488 DEBUG:tasks.cephadm:set 0 configs 2024-08-17T17:00:33.488 INFO:tasks.cephadm:Deploying OSDs... 2024-08-17T17:00:33.488 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:00:33.489 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T17:00:33.507 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T17:00:33.507 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-08-17 17:00:18.748816916 +0000 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:33.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T17:00:33.628 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-08-17T17:00:33.628 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-08-17T17:00:33.628 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000255947 s, 2.0 MB/s 2024-08-17T17:00:33.630 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T17:00:33.687 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 991 Links: 1 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-08-17 17:00:18.749816896 +0000 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.743 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T17:00:33.807 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-08-17T17:00:33.808 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-08-17T17:00:33.808 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000203614 s, 2.5 MB/s 2024-08-17T17:00:33.809 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T17:00:33.866 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2024-08-17T17:00:33.923 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T17:00:33.923 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:33.923 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 983 Links: 1 2024-08-17T17:00:33.923 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:33.924 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:33.924 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-08-17 17:00:18.749816896 +0000 2024-08-17T17:00:33.924 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.924 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.924 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-08-17 16:55:42.283561256 +0000 2024-08-17T17:00:33.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T17:00:33.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi175=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:33.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: Saving service mgr spec with placement smithi064=y;smithi175=x;count:2 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: Deploying daemon mgr.x on smithi175 2024-08-17T17:00:33.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[40964]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi175=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: Saving service mgr spec with placement smithi064=y;smithi175=x;count:2 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: Deploying daemon mgr.x on smithi175 2024-08-17T17:00:33.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:33 smithi064 ceph-mon[30574]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:33.989 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-08-17T17:00:33.989 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-08-17T17:00:33.989 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000203618 s, 2.5 MB/s 2024-08-17T17:00:33.990 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T17:00:34.047 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2024-08-17 17:00:18.750816875 +0000 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:34.104 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: 2024-08-17 16:55:42.282561276 +0000 2024-08-17T17:00:34.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi175=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: Saving service mgr spec with placement smithi064=y;smithi175=x;count:2 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: Deploying daemon mgr.x on smithi175 2024-08-17T17:00:34.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:33 smithi175 ceph-mon[34606]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:34.168 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2024-08-17T17:00:34.168 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2024-08-17T17:00:34.168 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000197265 s, 2.6 MB/s 2024-08-17T17:00:34.170 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T17:00:34.226 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T17:00:34.226 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T17:00:34.243 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T17:00:34.243 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_1 2024-08-17T17:00:34.298 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T17:00:34.298 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 968 Links: 1 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-08-17 17:00:10.211457939 +0000 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:34.299 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:34.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T17:00:34.363 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-08-17T17:00:34.364 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-08-17T17:00:34.364 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000217246 s, 2.4 MB/s 2024-08-17T17:00:34.365 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T17:00:34.421 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_2 2024-08-17T17:00:34.476 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T17:00:34.476 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:34.476 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 983 Links: 1 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-08-17 17:00:10.211457939 +0000 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.477 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T17:00:34.667 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-08-17T17:00:34.667 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-08-17T17:00:34.667 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000238182 s, 2.1 MB/s 2024-08-17T17:00:34.668 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T17:00:34.683 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_3 2024-08-17T17:00:34.739 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T17:00:34.739 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-08-17 17:00:34.620091732 +0000 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.740 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-08-17 16:55:41.961624806 +0000 2024-08-17T17:00:34.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T17:00:34.937 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-08-17T17:00:34.938 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-08-17T17:00:34.938 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000252895 s, 2.0 MB/s 2024-08-17T17:00:34.939 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T17:00:34.955 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_4 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-08-17 17:00:34.896087591 +0000 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:35.013 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-08-17 16:55:41.960624822 +0000 2024-08-17T17:00:35.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T17:00:35.076 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:34 smithi175 systemd[1]: Starting Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:00:35.080 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-08-17T17:00:35.080 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-08-17T17:00:35.080 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000220736 s, 2.3 MB/s 2024-08-17T17:00:35.081 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T17:00:35.138 INFO:tasks.cephadm:Deploying osd.0 on smithi064 with /dev/vg_nvme/lv_4... 2024-08-17T17:00:35.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T17:00:35.344 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:35.344 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 podman[36421]: 2024-08-17 17:00:35.076763432 +0000 UTC m=+0.017589939 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:00:35.344 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 podman[36421]: 2024-08-17 17:00:35.240015801 +0000 UTC m=+0.180842350 container create 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:00:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:35 smithi064 ceph-mon[40964]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:35 smithi064 ceph-mon[30574]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:35 smithi175 ceph-mon[34606]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 podman[36421]: 2024-08-17 17:00:35.434463634 +0000 UTC m=+0.375290140 container init 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 podman[36421]: 2024-08-17 17:00:35.437263201 +0000 UTC m=+0.378089707 container start 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 ceph-mgr[36477]: -- 172.21.15.175:0/461744495 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5598e59bed00 con 0x5598e5992800 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 bash[36421]: 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 systemd[1]: Started Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:00:35.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:35.557+0000 7f99f0f91040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:00:36.056 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:35.711+0000 7f99f0f91040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:00:36.056 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:35.781+0000 7f99f0f91040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:00:36.056 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:36.054+0000 7f99f0f91040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:00:36.370 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:36.113+0000 7f99f0f91040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:00:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:36.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:36.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:37.255 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.078+0000 7f99f0f91040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:00:37.255 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.197+0000 7f99f0f91040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:00:37.256 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.254+0000 7f99f0f91040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:00:37.526 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.465+0000 7f99f0f91040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:00:37.526 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.525+0000 7f99f0f91040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:37.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:37.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:00:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.771 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:00:37.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_4 2024-08-17T17:00:37.902 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.590+0000 7f99f0f91040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:00:37.902 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:37.733+0000 7f99f0f91040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:00:37.902 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.902 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.902 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:37.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:38.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:38.145+0000 7f99f0f91040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:00:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:38.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:38.452+0000 7f99f0f91040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:00:38.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:38.516+0000 7f99f0f91040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:00:38.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:38.661+0000 7f99f0f91040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:00:39.346 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:38.961+0000 7f99f0f91040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:00:39.346 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.021+0000 7f99f0f91040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:00:39.346 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.202+0000 7f99f0f91040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:00:39.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:39 smithi064 ceph-mon[30574]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:39.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:39 smithi064 ceph-mon[40964]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:39.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-mon[34606]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:39.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.349+0000 7f99f0f91040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T17:00:39.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.411+0000 7f99f0f91040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:00:39.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.492+0000 7f99f0f91040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:00:39.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:39.548+0000 7f99f0f91040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:00:40.347 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:40.066+0000 7f99f0f91040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:00:40.347 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:00:40.218+0000 7f99f0f91040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:00:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: Standby manager daemon x started 2024-08-17T17:00:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2921017325' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:41.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:00:41.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:41.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:41.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[30574]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:41.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:41 smithi064 ceph-mon[40964]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:00:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:41 smithi175 ceph-mon[34606]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:42.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1549760850' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]': finished 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[30574]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1549760850' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]': finished 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[40964]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:00:42.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:42 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:42 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1549760850' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:42 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]: dispatch 2024-08-17T17:00:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:42 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "88904c1f-eac1-41db-b79a-1a4967d1dae9"}]': finished 2024-08-17T17:00:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:42 smithi175 ceph-mon[34606]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:00:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:42 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:44.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:43 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/478157338' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:00:44.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:43 smithi175 ceph-mon[34606]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:43 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/478157338' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:00:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:43 smithi064 ceph-mon[30574]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:44.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:43 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/478157338' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:00:44.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:43 smithi064 ceph-mon[40964]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:45 smithi175 ceph-mon[34606]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:45.683 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:45 smithi064 ceph-mon[30574]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:45.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:45 smithi064 ceph-mon[40964]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:47.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:47 smithi064 ceph-mon[30574]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:47 smithi064 ceph-mon[40964]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:47 smithi175 ceph-mon[34606]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[40964]: Deploying daemon osd.0 on smithi064 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:48.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:48 smithi064 ceph-mon[30574]: Deploying daemon osd.0 on smithi064 2024-08-17T17:00:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:00:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:48 smithi175 ceph-mon[34606]: Deploying daemon osd.0 on smithi064 2024-08-17T17:00:49.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:49 smithi064 ceph-mon[30574]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:49.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:49 smithi064 ceph-mon[40964]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:49 smithi175 ceph-mon[34606]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:51 smithi175 ceph-mon[34606]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:51.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:51 smithi064 ceph-mon[30574]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:51.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:51 smithi064 ceph-mon[40964]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:53.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:53.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:53.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:53.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:53.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:53.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:54.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:54 smithi064 ceph-mon[30574]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:54.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:54 smithi064 ceph-mon[40964]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:54 smithi175 ceph-mon[34606]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:55 smithi064 ceph-mon[30574]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:55.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:55 smithi064 ceph-mon[40964]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:55 smithi175 ceph-mon[34606]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:56.494 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 0 on host 'smithi064' 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[40964]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:56 smithi064 ceph-mon[30574]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:56 smithi175 ceph-mon[34606]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:00:57.159 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.0.service 2024-08-17T17:00:57.161 INFO:tasks.cephadm:Deploying osd.1 on smithi064 with /dev/vg_nvme/lv_3... 2024-08-17T17:00:57.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T17:00:57.780 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.781 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:57 smithi175 ceph-mon[34606]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:57.872 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:57.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[40964]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:00:57.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:57.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:00:57.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:57.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:00:57.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-mon[30574]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:00:58.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[44957]: 2024-08-17T17:00:57.916+0000 7f7d4e5f2640 -1 osd.0 0 waiting for initial osdmap 2024-08-17T17:00:58.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:00:57 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[44957]: 2024-08-17T17:00:57.921+0000 7f7d46413640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' 2024-08-17T17:00:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:58 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' 2024-08-17T17:00:58.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: from='osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379]' entity='osd.0' 2024-08-17T17:00:58.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:58 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:01:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379] boot 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[30574]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379] boot 2024-08-17T17:01:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:01:00.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:01:00.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:00:59 smithi064 ceph-mon[40964]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: osd.0 [v2:172.21.15.64:6802/1608794379,v1:172.21.15.64:6803/1608794379] boot 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:01:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:00:59 smithi175 ceph-mon[34606]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:01.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.324 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:01.325 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:01:01.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_3 2024-08-17T17:01:02.098 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:02.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi064 2024-08-17T17:01:02.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi064 to 16064M 2024-08-17T17:01:02.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[30574]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi064 2024-08-17T17:01:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi064 to 16064M 2024-08-17T17:01:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:01 smithi064 ceph-mon[40964]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:01 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi064 2024-08-17T17:01:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:01 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi064 to 16064M 2024-08-17T17:01:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:01 smithi175 ceph-mon[34606]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:03.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:03 smithi064 ceph-mon[30574]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:03.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:03 smithi064 ceph-mon[40964]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:03 smithi175 ceph-mon[34606]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:04.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[30574]: from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:04.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:04.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:04.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:04.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[40964]: from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:04.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:04.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:04.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:04 smithi175 ceph-mon[34606]: from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:05.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:05 smithi064 ceph-mon[30574]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:05.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:05 smithi064 ceph-mon[40964]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:05 smithi175 ceph-mon[34606]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2732585693' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]': finished 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[30574]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2732585693' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]': finished 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[40964]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:01:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:06 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:06 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2732585693' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:06 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]: dispatch 2024-08-17T17:01:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:06 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d08169c-5914-4c23-acde-0607da1c3551"}]': finished 2024-08-17T17:01:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:06 smithi175 ceph-mon[34606]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:01:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:06 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:07 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/768299585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:07 smithi175 ceph-mon[34606]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:07 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/768299585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:07.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:07 smithi064 ceph-mon[30574]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:07 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/768299585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:07 smithi064 ceph-mon[40964]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:09 smithi175 ceph-mon[34606]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:09 smithi064 ceph-mon[30574]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:09.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:09 smithi064 ceph-mon[40964]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:11 smithi064 ceph-mon[30574]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:11 smithi064 ceph-mon[40964]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:11 smithi175 ceph-mon[34606]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:12.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:01:12.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:12.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[30574]: Deploying daemon osd.1 on smithi064 2024-08-17T17:01:12.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:01:12.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:12.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:12 smithi064 ceph-mon[40964]: Deploying daemon osd.1 on smithi064 2024-08-17T17:01:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:12 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:01:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:12 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:12 smithi175 ceph-mon[34606]: Deploying daemon osd.1 on smithi064 2024-08-17T17:01:13.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:13 smithi064 ceph-mon[40964]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:13 smithi064 ceph-mon[30574]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:13 smithi175 ceph-mon[34606]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:15 smithi064 ceph-mon[30574]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:15 smithi064 ceph-mon[40964]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:15 smithi175 ceph-mon[34606]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:17 smithi175 ceph-mon[34606]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:17 smithi064 ceph-mon[30574]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:18.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:17 smithi064 ceph-mon[40964]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:20.093 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 1 on host 'smithi064' 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:01:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:20.710 DEBUG:teuthology.orchestra.run.smithi064:osd.1> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.1.service 2024-08-17T17:01:20.713 INFO:tasks.cephadm:Deploying osd.2 on smithi064 with /dev/vg_nvme/lv_2... 2024-08-17T17:01:20.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T17:01:21.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[40964]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:01:21.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[40964]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:01:21.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:21.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[40964]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[40964]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[30574]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:01:21.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[30574]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:01:21.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:21.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[30574]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-mon[30574]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.439 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[50614]: 2024-08-17T17:01:21.244+0000 7f835bd9f640 -1 osd.1 0 waiting for initial osdmap 2024-08-17T17:01:21.439 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[50614]: 2024-08-17T17:01:21.246+0000 7f8357bc8640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:01:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:21 smithi175 ceph-mon[34606]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:01:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:21 smithi175 ceph-mon[34606]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:01:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:21 smithi175 ceph-mon[34606]: from='osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:21 smithi175 ceph-mon[34606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:21.884 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[30574]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[30574]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[30574]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[40964]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[40964]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:01:22.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:22 smithi064 ceph-mon[40964]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:22 smithi175 ceph-mon[34606]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:22 smithi175 ceph-mon[34606]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:01:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:22 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:22 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:22 smithi175 ceph-mon[34606]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:01:23.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:01:23.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067] boot 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067] boot 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:01:23.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:23 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: osd.1 [v2:172.21.15.64:6810/2757074067,v1:172.21.15.64:6811/2757074067] boot 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:01:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:23 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:01:24.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:01:24.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:24.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:24 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:01:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:24 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:25.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:25 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi064 2024-08-17T17:01:25.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:25 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi064 to 8032M 2024-08-17T17:01:25.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:25 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi064 2024-08-17T17:01:25.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:25 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi064 to 8032M 2024-08-17T17:01:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:01:25.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_2 2024-08-17T17:01:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:25 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi064 2024-08-17T17:01:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:25 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi064 to 8032M 2024-08-17T17:01:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:26 smithi064 ceph-mon[30574]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:26 smithi064 ceph-mon[40964]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:26 smithi175 ceph-mon[34606]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:27 smithi175 ceph-mon[34606]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:27 smithi064 ceph-mon[30574]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:27.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:27 smithi064 ceph-mon[40964]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:28.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[40964]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:28.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:28.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:28.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[30574]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:28.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:28 smithi175 ceph-mon[34606]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:28 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:28 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:28 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:28.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:28 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:29.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:29 smithi064 ceph-mon[30574]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:29.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:29 smithi064 ceph-mon[40964]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:29.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:29 smithi175 ceph-mon[34606]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:30.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]: dispatch 2024-08-17T17:01:30.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]': finished 2024-08-17T17:01:30.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[30574]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:01:30.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:30.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/445445720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:30.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]: dispatch 2024-08-17T17:01:30.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]': finished 2024-08-17T17:01:30.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[40964]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:01:30.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:30.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:30 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/445445720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:30 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]: dispatch 2024-08-17T17:01:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:30 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1167264512' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "870e8cbe-170a-40f5-8482-de1f6ef4061e"}]': finished 2024-08-17T17:01:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:30 smithi175 ceph-mon[34606]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:01:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:30 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:30 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/445445720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:31 smithi175 ceph-mon[34606]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:31.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:31 smithi064 ceph-mon[30574]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:31.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:31 smithi064 ceph-mon[40964]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:33 smithi175 ceph-mon[34606]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:33.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:33 smithi064 ceph-mon[30574]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:33.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:33 smithi064 ceph-mon[40964]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:35.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:35 smithi064 ceph-mon[30574]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:35.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:35 smithi064 ceph-mon[40964]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:35 smithi175 ceph-mon[34606]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[30574]: Deploying daemon osd.2 on smithi064 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:36 smithi064 ceph-mon[40964]: Deploying daemon osd.2 on smithi064 2024-08-17T17:01:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:01:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:36 smithi175 ceph-mon[34606]: Deploying daemon osd.2 on smithi064 2024-08-17T17:01:37.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:37 smithi064 ceph-mon[30574]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:37.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:37 smithi064 ceph-mon[40964]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:37 smithi175 ceph-mon[34606]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:39.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:39 smithi064 ceph-mon[30574]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:39.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:39 smithi064 ceph-mon[40964]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:39.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:39 smithi175 ceph-mon[34606]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:40 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:40 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:41 smithi175 ceph-mon[34606]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:41.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:41 smithi064 ceph-mon[30574]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:41.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:41 smithi064 ceph-mon[40964]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:43.858 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 2 on host 'smithi064' 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:44.455 DEBUG:teuthology.orchestra.run.smithi064:osd.2> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.2.service 2024-08-17T17:01:44.457 INFO:tasks.cephadm:Deploying osd.3 on smithi064 with /dev/vg_nvme/lv_1... 2024-08-17T17:01:44.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T17:01:45.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[30574]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:01:45.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[30574]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:01:45.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:45.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[30574]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:45.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[40964]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:01:45.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[40964]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:01:45.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:45.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-mon[40964]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:45.267 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[56341]: 2024-08-17T17:01:45.129+0000 7f1d28919640 -1 osd.2 0 waiting for initial osdmap 2024-08-17T17:01:45.267 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:45 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[56341]: 2024-08-17T17:01:45.132+0000 7f1d24742640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:01:45.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:45 smithi175 ceph-mon[34606]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:01:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:45 smithi175 ceph-mon[34606]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:01:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:45 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:45 smithi175 ceph-mon[34606]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:01:45.683 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:46 smithi175 ceph-mon[34606]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:46 smithi175 ceph-mon[34606]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:01:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:46 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:46 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:46 smithi175 ceph-mon[34606]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[40964]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[40964]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:01:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[40964]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[30574]: from='osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:01:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[30574]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:01:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:46 smithi064 ceph-mon[30574]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T17:01:47.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:01:47.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:01:47.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[40964]: osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719] boot 2024-08-17T17:01:47.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[40964]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[30574]: osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719] boot 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[30574]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:01:47.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:47 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:47 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:01:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:47 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:01:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:47 smithi175 ceph-mon[34606]: osd.2 [v2:172.21.15.64:6818/1249488719,v1:172.21.15.64:6819/1249488719] boot 2024-08-17T17:01:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:47 smithi175 ceph-mon[34606]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:01:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:47 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:01:48.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: pgmap v74: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:48.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: pgmap v74: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:48.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:48 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: pgmap v74: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:01:48.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:48.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:48.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:48.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:48 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:49 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi064 2024-08-17T17:01:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:49 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi064 to 5354M 2024-08-17T17:01:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:49 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:49 smithi175 ceph-mon[34606]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:01:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:49 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi064 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi064 to 5354M 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[30574]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi064 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi064 to 5354M 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[40964]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:01:49.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:49 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:01:49.466 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:01:49.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi064:vg_nvme/lv_1 2024-08-17T17:01:49.710 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[30574]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:01:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[30574]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[40964]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:01:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:50 smithi064 ceph-mon[40964]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:50.438 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60409]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:50.439 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60409]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:50.439 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60409]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:50.439 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60409]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:50 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:01:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:50 smithi175 ceph-mon[34606]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:01:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:50 smithi175 ceph-mon[34606]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:50.844 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60420]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:50.845 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60420]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:50.845 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60420]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:50.845 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:01:50 smithi064 sudo[60420]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:51.139 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60443]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:51.139 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60443]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:51.139 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60443]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:51.140 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60443]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:51 smithi064 ceph-mon[30574]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T17:01:51.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:51 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:51 smithi064 ceph-mon[40964]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T17:01:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:51 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:51 smithi175 ceph-mon[34606]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T17:01:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:51 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:51.927 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60471]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:51.927 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60471]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:51.927 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60471]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:51.927 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60471]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:52.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[30574]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:52.178 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60594]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:52.178 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60594]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:52.178 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:51 smithi064 sudo[60594]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:52.178 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60594]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:52.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[40964]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:52.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:52.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:52.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:52 smithi175 ceph-mon[34606]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:01:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:01:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:52.897 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60622]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:52.897 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60622]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:52.897 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60622]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:52.897 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60622]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:53.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60633]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:53.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60633]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:53.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60633]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:53.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:52 smithi064 sudo[60633]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:53.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 sudo[60665]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:53.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 sudo[60665]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:53.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 sudo[60665]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 sudo[60665]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[30574]: from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[30574]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[40964]: from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[40964]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:01:53.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:53 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:53.633 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 ceph-mon[34606]: from='client.14295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.633 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 ceph-mon[34606]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:01:53.634 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:53.634 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37154]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:53.634 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37154]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:53.634 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37154]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:53.634 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37154]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37157]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37157]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37157]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:53 smithi175 sudo[37157]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60708]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:01:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60708]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60708]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60708]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]: dispatch 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60711]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60711]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60711]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 sudo[60711]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:55.054 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:54 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]: dispatch 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:01:55.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:54 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]: dispatch 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]': finished 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:55 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3348766954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:56.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:56.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:56.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]': finished 2024-08-17T17:01:56.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3348766954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2671168329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bad72588-3314-4260-ab5d-2b909812b54b"}]': finished 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:55 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3348766954' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:01:57.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:57 smithi064 ceph-mon[30574]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:57.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:57 smithi064 ceph-mon[40964]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:57 smithi175 ceph-mon[34606]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:01:59.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:01:59 smithi064 ceph-mon[30574]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T17:01:59.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:01:59 smithi064 ceph-mon[40964]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T17:01:59.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:01:59 smithi175 ceph-mon[34606]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T17:02:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:02:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:02:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:01.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:02:01.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:02.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:01 smithi064 ceph-mon[30574]: Deploying daemon osd.3 on smithi064 2024-08-17T17:02:02.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:01 smithi064 ceph-mon[30574]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T17:02:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:01 smithi064 ceph-mon[40964]: Deploying daemon osd.3 on smithi064 2024-08-17T17:02:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:01 smithi064 ceph-mon[40964]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T17:02:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:01 smithi175 ceph-mon[34606]: Deploying daemon osd.3 on smithi064 2024-08-17T17:02:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:01 smithi175 ceph-mon[34606]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T17:02:03.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:03 smithi064 ceph-mon[30574]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T17:02:03.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:03 smithi064 ceph-mon[40964]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T17:02:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:03 smithi175 ceph-mon[34606]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T17:02:05.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:05 smithi064 ceph-mon[30574]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 75 KiB/s wr, 5 op/s 2024-08-17T17:02:05.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:05 smithi064 ceph-mon[40964]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 75 KiB/s wr, 5 op/s 2024-08-17T17:02:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:05 smithi175 ceph-mon[34606]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 75 KiB/s wr, 5 op/s 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:06.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:06 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:06 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:06 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:06 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:07.572 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:07 smithi064 ceph-mon[30574]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:07.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:07 smithi064 ceph-mon[40964]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:07 smithi175 ceph-mon[34606]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:09.080 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 3 on host 'smithi064' 2024-08-17T17:02:09.687 DEBUG:teuthology.orchestra.run.smithi064:osd.3> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.3.service 2024-08-17T17:02:09.689 INFO:tasks.cephadm:Deploying osd.4 on smithi175 with /dev/vg_nvme/lv_4... 2024-08-17T17:02:09.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T17:02:09.899 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:10.355 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:10 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:10.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:10 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:11.139 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[30574]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[30574]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[30574]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[30574]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[40964]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[40964]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[40964]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-mon[40964]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.329 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:11 smithi175 ceph-mon[34606]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:02:11.329 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:11 smithi175 ceph-mon[34606]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:02:11.329 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:11 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:11.329 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:11 smithi175 ceph-mon[34606]: from='osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.329 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:11 smithi175 ceph-mon[34606]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2024-08-17T17:02:11.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[62361]: 2024-08-17T17:02:11.528+0000 7f1e7c662640 -1 osd.3 0 waiting for initial osdmap 2024-08-17T17:02:11.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:02:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[62361]: 2024-08-17T17:02:11.532+0000 7f1e74483640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:02:12.270 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:02:12.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_4 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:02:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:12 smithi175 ceph-mon[34606]: from='osd.3 ' entity='osd.3' 2024-08-17T17:02:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:02:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T17:02:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[30574]: from='osd.3 ' entity='osd.3' 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T17:02:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:12 smithi064 ceph-mon[40964]: from='osd.3 ' entity='osd.3' 2024-08-17T17:02:12.510 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726] boot 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726] boot 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: osd.3 [v2:172.21.15.64:6826/495130726,v1:172.21.15.64:6827/495130726] boot 2024-08-17T17:02:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:13.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:13.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:13.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:14 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi064 2024-08-17T17:02:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:14 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi064 to 4016M 2024-08-17T17:02:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:14 smithi175 ceph-mon[34606]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi064 2024-08-17T17:02:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi064 to 4016M 2024-08-17T17:02:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[30574]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi064 2024-08-17T17:02:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi064 to 4016M 2024-08-17T17:02:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:14 smithi064 ceph-mon[40964]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:15 smithi175 ceph-mon[34606]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T17:02:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:15 smithi175 ceph-mon[34606]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:15 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:15 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:15 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[30574]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T17:02:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[30574]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[40964]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T17:02:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[40964]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:15 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:16 smithi175 ceph-mon[34606]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:16 smithi064 ceph-mon[30574]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:16 smithi064 ceph-mon[40964]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/800323024' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]': finished 2024-08-17T17:02:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:02:17.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:17.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:17 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/2431721538' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/800323024' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]': finished 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/2431721538' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/800323024' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]: dispatch 2024-08-17T17:02:17.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d8a51704-ce10-47b2-806f-ee7a6c85aab0"}]': finished 2024-08-17T17:02:17.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:02:17.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:17.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:17 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/2431721538' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:18.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:18 smithi175 ceph-mon[34606]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:18 smithi064 ceph-mon[30574]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:18 smithi064 ceph-mon[40964]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:19.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:19 smithi175 ceph-mon[34606]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:19 smithi064 ceph-mon[30574]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:19 smithi064 ceph-mon[40964]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:21.492 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:21 smithi175 ceph-mon[34606]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:21.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:21 smithi064 ceph-mon[40964]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:21 smithi064 ceph-mon[30574]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:22 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:02:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:22 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:22 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:02:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:22 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:22 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:02:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:22 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:23 smithi175 ceph-mon[34606]: Deploying daemon osd.4 on smithi175 2024-08-17T17:02:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:23 smithi175 ceph-mon[34606]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:23 smithi064 ceph-mon[30574]: Deploying daemon osd.4 on smithi175 2024-08-17T17:02:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:23 smithi064 ceph-mon[30574]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:23 smithi064 ceph-mon[40964]: Deploying daemon osd.4 on smithi175 2024-08-17T17:02:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:23 smithi064 ceph-mon[40964]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:25.646 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:25 smithi175 ceph-mon[34606]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:25 smithi064 ceph-mon[30574]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:25.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:25 smithi064 ceph-mon[40964]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:27.610 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:27.610 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:27.610 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:28.528 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:28 smithi175 ceph-mon[34606]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:28 smithi064 ceph-mon[30574]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:28 smithi064 ceph-mon[40964]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:29.555 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:29 smithi175 ceph-mon[34606]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:29.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:29 smithi064 ceph-mon[30574]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:29.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:29 smithi064 ceph-mon[40964]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:30.271 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 4 on host 'smithi175' 2024-08-17T17:02:30.850 DEBUG:teuthology.orchestra.run.smithi175:osd.4> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.4.service 2024-08-17T17:02:30.852 INFO:tasks.cephadm:Deploying osd.5 on smithi175 with /dev/vg_nvme/lv_3... 2024-08-17T17:02:30.852 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T17:02:31.098 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:31.478 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.478 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.478 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.478 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.478 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.479 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:31 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:02:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:31.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:31 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:32.386 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:02:32.386 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:02:32.386 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:32.386 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.386 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.387 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-mon[34606]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:32.387 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:02:32.280+0000 7fc4ee7db640 -1 osd.4 0 waiting for initial osdmap 2024-08-17T17:02:32.387 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:02:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:02:32.283+0000 7fc4ea604640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[30574]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: from='osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:32.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:32 smithi064 ceph-mon[40964]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T17:02:33.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[30574]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[30574]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[40964]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[40964]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:33 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:33 smithi175 ceph-mon[34606]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:33 smithi175 ceph-mon[34606]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T17:02:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:33 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:33.780 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:02:33.802 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_3 2024-08-17T17:02:34.490 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:02:34.490 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:02:34.491 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260] boot 2024-08-17T17:02:34.491 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:02:34.491 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:34.491 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:34.491 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:34 smithi175 ceph-mon[34606]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:02:34.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260] boot 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[40964]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:02:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: osd.4 [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260] boot 2024-08-17T17:02:34.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:02:34.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:34.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:34.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:34 smithi064 ceph-mon[30574]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:02:34.734 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:36.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:02:36.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:36.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:02:36.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.620 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:36 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:36.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:36 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:37.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi175 2024-08-17T17:02:37.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 17088M 2024-08-17T17:02:37.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T17:02:37.575 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:37.575 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:37.575 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:37 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi175 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 17088M 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi175 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 17088M 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:02:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:37 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:38.543 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:38 smithi175 ceph-mon[34606]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.543 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:38 smithi175 ceph-mon[34606]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 134 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:38 smithi064 ceph-mon[30574]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:38 smithi064 ceph-mon[30574]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 134 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:38 smithi064 ceph-mon[40964]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:38 smithi064 ceph-mon[40964]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 134 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:39.647 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:39 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/795921134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:39 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:39 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]': finished 2024-08-17T17:02:39.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:39 smithi175 ceph-mon[34606]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:02:39.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:39 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:39.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/795921134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]': finished 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[30574]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/795921134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]: dispatch 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1ebe3c8b-daa3-49aa-a503-4671f0f97c95"}]': finished 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[40964]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:02:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:39 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:40 smithi175 ceph-mon[34606]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:02:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:40 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/1268771923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:40 smithi064 ceph-mon[30574]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:02:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:40 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/1268771923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:40 smithi064 ceph-mon[40964]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:02:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:40 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/1268771923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:02:42.465 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:42 smithi175 ceph-mon[34606]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:02:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:42 smithi064 ceph-mon[30574]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:02:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:42 smithi064 ceph-mon[40964]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:02:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:44 smithi175 ceph-mon[34606]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:02:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:44 smithi064 ceph-mon[30574]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:02:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:44 smithi064 ceph-mon[40964]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:02:45.396 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:45 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:02:45.396 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:45 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:45.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:45 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:02:45.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:45 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:45.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:45 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:02:45.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:45 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:46.568 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:46 smithi175 ceph-mon[34606]: Deploying daemon osd.5 on smithi175 2024-08-17T17:02:46.568 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:46 smithi175 ceph-mon[34606]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:02:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:46 smithi064 ceph-mon[30574]: Deploying daemon osd.5 on smithi175 2024-08-17T17:02:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:46 smithi064 ceph-mon[30574]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:02:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:46 smithi064 ceph-mon[40964]: Deploying daemon osd.5 on smithi175 2024-08-17T17:02:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:46 smithi064 ceph-mon[40964]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:02:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:48 smithi175 ceph-mon[34606]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:02:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:48 smithi064 ceph-mon[30574]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:02:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:48 smithi064 ceph-mon[40964]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:02:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:49 smithi175 ceph-mon[34606]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:49 smithi064 ceph-mon[30574]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:49.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:49 smithi064 ceph-mon[40964]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:50.475 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:50 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:50.476 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:50 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:50.476 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:50 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:50.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:50 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:51.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:51 smithi175 ceph-mon[34606]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:51 smithi064 ceph-mon[30574]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:51 smithi064 ceph-mon[40964]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:53.021 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 5 on host 'smithi175' 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:53.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:53 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:53 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:53.627 DEBUG:teuthology.orchestra.run.smithi175:osd.5> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T17:02:53.629 INFO:tasks.cephadm:Deploying osd.6 on smithi175 with /dev/vg_nvme/lv_2... 2024-08-17T17:02:53.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-mon[34606]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[30574]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: from='osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:54 smithi064 ceph-mon[40964]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:02:54.715 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:54.975 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[45407]: 2024-08-17T17:02:54.782+0000 7f2ab8a22640 -1 osd.5 0 waiting for initial osdmap 2024-08-17T17:02:54.976 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:02:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[45407]: 2024-08-17T17:02:54.787+0000 7f2ab0843640 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: from='osd.5 ' entity='osd.5' 2024-08-17T17:02:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:55 smithi175 ceph-mon[34606]: pgmap v127: 1 pgs: 1 unknown; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: from='osd.5 ' entity='osd.5' 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[30574]: pgmap v127: 1 pgs: 1 unknown; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: from='osd.5 ' entity='osd.5' 2024-08-17T17:02:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:55 smithi064 ceph-mon[40964]: pgmap v127: 1 pgs: 1 unknown; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320] boot 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:56 smithi175 ceph-mon[34606]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320] boot 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[40964]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: osd.5 [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320] boot 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:02:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:56 smithi064 ceph-mon[30574]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi175 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 8544M 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:57 smithi175 ceph-mon[34606]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:02:58.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi175 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 8544M 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[30574]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi175 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 8544M 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:02:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:57 smithi064 ceph-mon[40964]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:02:58.385 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:02:58.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_2 2024-08-17T17:02:58.604 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:02:59.068 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:58 smithi175 ceph-mon[34606]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:02:59.068 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:58 smithi175 ceph-mon[34606]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T17:02:59.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:58 smithi064 ceph-mon[30574]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:02:59.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:58 smithi064 ceph-mon[30574]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T17:02:59.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:58 smithi064 ceph-mon[40964]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:02:59.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:58 smithi064 ceph-mon[40964]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T17:02:59.989 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:02:59 smithi175 ceph-mon[34606]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:02:59 smithi064 ceph-mon[30574]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:02:59 smithi064 ceph-mon[40964]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:01.044 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:01.044 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:01.044 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:00 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:01.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:01.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:00 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:01 smithi175 ceph-mon[34606]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:01.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:01 smithi175 ceph-mon[34606]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-08-17T17:03:02.089 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:01 smithi064 ceph-mon[30574]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:02.089 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:01 smithi064 ceph-mon[30574]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-08-17T17:03:02.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:01 smithi064 ceph-mon[40964]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:02.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:01 smithi064 ceph-mon[40964]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-08-17T17:03:03.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:02 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/1902086761' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:02 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:02 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]': finished 2024-08-17T17:03:03.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:02 smithi175 ceph-mon[34606]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:03:03.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:02 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/1902086761' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]': finished 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[30574]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/1902086761' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]: dispatch 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bdf0ddc-f45b-4db6-9a7a-1769117b5b83"}]': finished 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[40964]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:03:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:02 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:03 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/3509701895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:03 smithi175 ceph-mon[34606]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:03 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/3509701895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:04.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:03 smithi064 ceph-mon[30574]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:03 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/3509701895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:03 smithi064 ceph-mon[40964]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:05 smithi175 ceph-mon[34606]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:03:05.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:05 smithi064 ceph-mon[30574]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:03:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:05 smithi064 ceph-mon[40964]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:03:07.655 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:07 smithi175 ceph-mon[34606]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:03:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:07 smithi064 ceph-mon[30574]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:03:07.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:07 smithi064 ceph-mon[40964]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:03:08.880 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:08 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:03:08.880 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:08 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:08 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:03:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:08 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:08 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:03:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:08 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:09.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:09 smithi175 ceph-mon[34606]: Deploying daemon osd.6 on smithi175 2024-08-17T17:03:09.740 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:09 smithi175 ceph-mon[34606]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:09 smithi064 ceph-mon[30574]: Deploying daemon osd.6 on smithi175 2024-08-17T17:03:09.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:09 smithi064 ceph-mon[30574]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:09 smithi064 ceph-mon[40964]: Deploying daemon osd.6 on smithi175 2024-08-17T17:03:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:09 smithi064 ceph-mon[40964]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:11.607 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:11 smithi175 ceph-mon[34606]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:11 smithi064 ceph-mon[30574]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:11 smithi064 ceph-mon[40964]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:13 smithi175 ceph-mon[34606]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:13 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:13 smithi064 ceph-mon[30574]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:13 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:13 smithi064 ceph-mon[40964]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:13 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:14 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:14 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:14 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:14 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:14 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:14 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:15 smithi175 ceph-mon[34606]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:15 smithi064 ceph-mon[30574]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:15 smithi064 ceph-mon[40964]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:16.789 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 6 on host 'smithi175' 2024-08-17T17:03:17.091 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.092 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:16 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.405 DEBUG:teuthology.orchestra.run.smithi175:osd.6> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T17:03:17.407 INFO:tasks.cephadm:Deploying osd.7 on smithi175 with /dev/vg_nvme/lv_1... 2024-08-17T17:03:17.407 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T17:03:17.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:17.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:16 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:03:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:03:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:17 smithi175 ceph-mon[34606]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:03:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[30574]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: from='osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:17 smithi064 ceph-mon[40964]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T17:03:18.760 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:03:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[51336]: 2024-08-17T17:03:18.448+0000 7f7da5e69640 -1 osd.6 0 waiting for initial osdmap 2024-08-17T17:03:18.760 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:03:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[51336]: 2024-08-17T17:03:18.455+0000 7f7d9dc8a640 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:03:19.087 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:19 smithi175 ceph-mon[34606]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:19.087 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:19 smithi175 ceph-mon[34606]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T17:03:19.087 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:19 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.087 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:19 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.087 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:19 smithi175 ceph-mon[34606]: from='osd.6 ' entity='osd.6' 2024-08-17T17:03:19.094 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:18 smithi064 ceph-mon[30574]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:18 smithi064 ceph-mon[30574]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:18 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:18 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:18 smithi064 ceph-mon[30574]: from='osd.6 ' entity='osd.6' 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:19 smithi064 ceph-mon[40964]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:19 smithi064 ceph-mon[40964]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T17:03:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:19 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:19.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:19 smithi064 ceph-mon[40964]: from='osd.6 ' entity='osd.6' 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961] boot 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:20 smithi175 ceph-mon[34606]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961] boot 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[30574]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:03:20.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: osd.6 [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961] boot 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:20.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:20 smithi064 ceph-mon[40964]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:03:22.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi175 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 5696M 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:03:22.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[30574]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi175 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 5696M 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:22.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:03:22.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:21 smithi064 ceph-mon[40964]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi175 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:22.263 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 5696M 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:03:22.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:21 smithi175 ceph-mon[34606]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:22.266 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-08-17T17:03:22.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi175:vg_nvme/lv_1 2024-08-17T17:03:22.491 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:03:23.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:23 smithi175 ceph-mon[34606]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:03:23.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:23 smithi064 ceph-mon[30574]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:03:23.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:23 smithi064 ceph-mon[40964]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:03:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:24 smithi175 ceph-mon[34606]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T17:03:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:24 smithi175 ceph-mon[34606]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:24 smithi064 ceph-mon[30574]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T17:03:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:24 smithi064 ceph-mon[30574]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:24 smithi064 ceph-mon[40964]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T17:03:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:24 smithi064 ceph-mon[40964]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:25 smithi175 ceph-mon[34606]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:25 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[30574]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[40964]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:03:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:25 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:26 smithi175 ceph-mon[34606]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-08-17T17:03:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:26 smithi064 ceph-mon[30574]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-08-17T17:03:26.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:26 smithi064 ceph-mon[40964]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/15734544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]': finished 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:27 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/2643293311' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/15734544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]': finished 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/2643293311' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/15734544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]: dispatch 2024-08-17T17:03:27.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6aec1a-de13-4aa2-a0cf-5533286fcce6"}]': finished 2024-08-17T17:03:27.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:03:27.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:27.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:27 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/2643293311' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:03:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:28 smithi175 ceph-mon[34606]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:28 smithi064 ceph-mon[30574]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:28 smithi064 ceph-mon[40964]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:29.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:29 smithi175 ceph-mon[34606]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-08-17T17:03:29.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:29 smithi064 ceph-mon[30574]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-08-17T17:03:29.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:29 smithi064 ceph-mon[40964]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-08-17T17:03:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:31 smithi175 ceph-mon[34606]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:03:31.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:31 smithi064 ceph-mon[30574]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:03:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:31 smithi064 ceph-mon[40964]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:03:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:03:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:32 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:03:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:32 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:32.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:03:32.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:32 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:33.763 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:33 smithi175 ceph-mon[34606]: Deploying daemon osd.7 on smithi175 2024-08-17T17:03:33.763 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:33 smithi175 ceph-mon[34606]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:33.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:33 smithi064 ceph-mon[30574]: Deploying daemon osd.7 on smithi175 2024-08-17T17:03:33.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:33 smithi064 ceph-mon[30574]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:33 smithi064 ceph-mon[40964]: Deploying daemon osd.7 on smithi175 2024-08-17T17:03:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:33 smithi064 ceph-mon[40964]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:03:35.580 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:35 smithi175 ceph-mon[34606]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:35.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:35 smithi064 ceph-mon[30574]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:35 smithi064 ceph-mon[40964]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:37 smithi175 ceph-mon[34606]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:37 smithi064 ceph-mon[30574]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:37.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:37 smithi064 ceph-mon[40964]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:38 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:38 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:38 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:38 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:39 smithi175 ceph-mon[34606]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:39.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:39 smithi064 ceph-mon[30574]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:39.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:39 smithi064 ceph-mon[40964]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:40.812 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 7 on host 'smithi175' 2024-08-17T17:03:41.397 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.398 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:41 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.419 DEBUG:teuthology.orchestra.run.smithi175:osd.7> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:03:41.421 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-17T17:03:41.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T17:03:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:41 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:41.647 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.320 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-mon[34606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[40964]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:03:42.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:42.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: from='osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:42 smithi064 ceph-mon[30574]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-08-17T17:03:42.621 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[56879]: 2024-08-17T17:03:42.494+0000 7ff5a8c78640 -1 osd.7 0 waiting for initial osdmap 2024-08-17T17:03:42.621 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:03:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[56879]: 2024-08-17T17:03:42.500+0000 7ff5a4aa1640 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:03:43.508 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:03:43.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[30574]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:43.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[30574]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T17:03:43.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[30574]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[40964]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[40964]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:43 smithi064 ceph-mon[40964]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:43.877 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:43 smithi175 ceph-mon[34606]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-08-17T17:03:43.877 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:43 smithi175 ceph-mon[34606]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T17:03:43.877 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:43 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.877 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:43 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:43.877 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:43 smithi175 ceph-mon[34606]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T17:03:44.132 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1723914223,"num_in_osds":8,"osd_in_since":1723914206,"num_remapped_pgs":1} 2024-08-17T17:03:44.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:03:44.387 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:44.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: purged_snaps scrub starts 2024-08-17T17:03:44.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: purged_snaps scrub ok 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871] boot 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/463707644' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: Detected new or changed devices on smithi175 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 4272M 2024-08-17T17:03:44.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: purged_snaps scrub starts 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: purged_snaps scrub ok 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871] boot 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/463707644' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: Detected new or changed devices on smithi175 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 4272M 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:44.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:44 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: purged_snaps scrub starts 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: purged_snaps scrub ok 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: osd.7 [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871] boot 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/463707644' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: Detected new or changed devices on smithi175 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 4272M 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:44.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:44 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:45.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:45 smithi064 ceph-mon[30574]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:03:45.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:45 smithi064 ceph-mon[30574]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:45.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:45 smithi064 ceph-mon[40964]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:03:45.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:45 smithi064 ceph-mon[40964]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:45 smithi175 ceph-mon[34606]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:03:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:45 smithi175 ceph-mon[34606]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:46.170 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:03:46.170 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":58,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:03:45.716583+0000","last_up_change":"2024-08-17T17:03:43.486679+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6803","nonce":1608794379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6805","nonce":1608794379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6809","nonce":1608794379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6807","nonce":1608794379}]},"public_addr":"172.21.15.64:6803/1608794379","cluster_addr":"172.21.15.64:6805/1608794379","heartbeat_back_addr":"172.21.15.64:6809/1608794379","heartbeat_front_addr":"172.21.15.64:6807/1608794379","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6811","nonce":2757074067}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6813","nonce":2757074067}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6817","nonce":2757074067}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6815","nonce":2757074067}]},"public_addr":"172.21.15.64:6811/2757074067","cluster_addr":"172.21.15.64:6813/2757074067","heartbeat_back_addr":"172.21.15.64:6817/2757074067","heartbeat_front_addr":"172.21.15.64:6815/2757074067","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6819","nonce":1249488719}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6821","nonce":1249488719}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6825","nonce":1249488719}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6823","nonce":1249488719}]},"public_addr":"172.21.15.64:6819/1249488719","cluster_addr":"172.21.15.64:6821/1249488719","heartbeat_back_addr":"172.21.15.64:6825/1249488719","heartbeat_front_addr":"172.21.15.64:6823/1249488719","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6827","nonce":495130726}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6829","nonce":495130726}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6833","nonce":495130726}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6831","nonce":495130726}]},"public_addr":"172.21.15.64:6827/495130726","cluster_addr":"172.21.15.64:6829/495130726","heartbeat_back_addr":"172.21.15.64:6833/495130726","heartbeat_front_addr":"172.21.15.64:6831/495130726","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:03:46.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[30574]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:03:46.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[30574]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:03:46.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/681208293' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:03:46.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[40964]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:03:46.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[40964]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:03:46.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:46 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/681208293' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:03:46.805 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-17T17:01:47.759503+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-08-17T17:03:46.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-17T17:03:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:46 smithi175 ceph-mon[34606]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:03:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:46 smithi175 ceph-mon[34606]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:03:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:46 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/681208293' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:03:47.014 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:48.078 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:47 smithi175 ceph-mon[34606]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:03:48.078 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:47 smithi175 ceph-mon[34606]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:48.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:47 smithi064 ceph-mon[30574]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:03:48.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:47 smithi064 ceph-mon[30574]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:48.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:47 smithi064 ceph-mon[40964]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:03:48.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:47 smithi064 ceph-mon[40964]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:48.778 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2024-08-17T17:03:49.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:48 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1686425297' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:03:49.139 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:48 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1686425297' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:03:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:48 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1686425297' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:03:49.431 INFO:tasks.cephadm:Adding prometheus.a on smithi175 2024-08-17T17:03:49.432 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply prometheus '1;smithi175=a' 2024-08-17T17:03:49.637 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:03:50.089 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:49 smithi175 ceph-mon[34606]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-17T17:03:50.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:49 smithi064 ceph-mon[30574]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-17T17:03:50.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:49 smithi064 ceph-mon[40964]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-17T17:03:51.412 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled prometheus update... 2024-08-17T17:03:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:51 smithi175 ceph-mon[34606]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:51 smithi064 ceph-mon[30574]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:51.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:51 smithi064 ceph-mon[40964]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:03:52.072 DEBUG:teuthology.orchestra.run.smithi175:prometheus.a> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service 2024-08-17T17:03:52.075 INFO:tasks.cephadm:Adding node-exporter.a on smithi064 2024-08-17T17:03:52.075 INFO:tasks.cephadm:Adding node-exporter.b on smithi175 2024-08-17T17:03:52.075 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply node-exporter '2;smithi064=a;smithi175=b' 2024-08-17T17:03:52.337 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: Saving service prometheus spec with placement smithi175=a;count:1 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:52 smithi175 ceph-mon[34606]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:03:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: Saving service prometheus spec with placement smithi175=a;count:1 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[30574]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi175=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: Saving service prometheus spec with placement smithi175=a;count:1 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:52 smithi064 ceph-mon[40964]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:03:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:53 smithi175 ceph-mon[34606]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-17T17:03:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:53 smithi064 ceph-mon[30574]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-17T17:03:53.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:53 smithi064 ceph-mon[40964]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-17T17:03:54.574 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled node-exporter update... 2024-08-17T17:03:55.593 DEBUG:teuthology.orchestra.run.smithi064:node-exporter.a> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service 2024-08-17T17:03:55.596 DEBUG:teuthology.orchestra.run.smithi175:node-exporter.b> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service 2024-08-17T17:03:55.599 INFO:tasks.cephadm:Adding alertmanager.a on smithi064 2024-08-17T17:03:55.599 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply alertmanager '1;smithi064=a' 2024-08-17T17:03:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:55 smithi175 ceph-mon[34606]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:55 smithi175 ceph-mon[34606]: Saving service node-exporter spec with placement smithi064=a;smithi175=b;count:2 2024-08-17T17:03:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:55 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:55 smithi175 ceph-mon[34606]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-08-17T17:03:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[30574]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[30574]: Saving service node-exporter spec with placement smithi064=a;smithi175=b;count:2 2024-08-17T17:03:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[30574]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-08-17T17:03:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[40964]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi175=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[40964]: Saving service node-exporter spec with placement smithi064=a;smithi175=b;count:2 2024-08-17T17:03:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:03:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:55 smithi064 ceph-mon[40964]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-08-17T17:03:57.612 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:03:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:57 smithi175 ceph-mon[34606]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-08-17T17:03:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:57 smithi064 ceph-mon[30574]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-08-17T17:03:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:57 smithi064 ceph-mon[40964]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-08-17T17:03:59.508 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:03:59 smithi175 ceph-mon[34606]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:03:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:03:59 smithi064 ceph-mon[30574]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:03:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:03:59 smithi064 ceph-mon[40964]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:04:01.636 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:01 smithi175 ceph-mon[34606]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:01 smithi064 ceph-mon[30574]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:01 smithi064 ceph-mon[40964]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:03.232 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:03 smithi175 systemd[1]: Starting Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:04:03.232 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled alertmanager update... 2024-08-17T17:04:03.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:03 smithi175 podman[61410]: 2024-08-17 17:04:03.263805843 +0000 UTC m=+0.107097982 container create 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:03.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:03 smithi175 podman[61410]: 2024-08-17 17:04:03.175488952 +0000 UTC m=+0.018781104 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:04:03.957 DEBUG:teuthology.orchestra.run.smithi064:alertmanager.a> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service 2024-08-17T17:04:03.960 INFO:tasks.cephadm:Adding grafana.a on smithi175 2024-08-17T17:04:03.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph orch apply grafana '1;smithi175=a' 2024-08-17T17:04:04.196 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:04:04.233 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 podman[61410]: 2024-08-17 17:04:04.060369387 +0000 UTC m=+0.903661526 container init 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 podman[61410]: 2024-08-17 17:04:04.063663136 +0000 UTC m=+0.906955278 container start 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.087Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.087Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.087Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi175 (none))" 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.087Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.087Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.089Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.090Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.091Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.091Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:04:04.234 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.092Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.092Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.525µs 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.092Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.093Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.093Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.743µs wal_replay_duration=213.621µs wbl_replay_duration=117ns total_replay_duration=256.243µs 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.094Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.094Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.094Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.115Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.903172ms db_storage=854ns remote_storage=1.075µs web_handler=400ns query_engine=638ns scrape=444.024µs scrape_sd=76.147µs notify=647ns notify_sd=1.276µs rules=20.101126ms tracing=7.446µs 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.115Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:04:04.115Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 bash[61410]: 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 2024-08-17T17:04:04.235 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:04:04 smithi175 systemd[1]: Started Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:04:04.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: Saving service alertmanager spec with placement smithi064=a;count:1 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: Saving service alertmanager spec with placement smithi064=a;count:1 2024-08-17T17:04:04.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:04.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:04 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: Saving service alertmanager spec with placement smithi064=a;count:1 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.556 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' 2024-08-17T17:04:04.557 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:04 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:04:05.500 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: ignoring --setuser ceph since I am not root 2024-08-17T17:04:05.500 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: ignoring --setgroup ceph since I am not root 2024-08-17T17:04:05.500 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-mgr[36477]: -- 172.21.15.175:0/2127763069 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d14a916d00 con 0x55d14a8eb000 2024-08-17T17:04:05.500 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:05.349+0000 7fb2a2f4f040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:04:05.500 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:05.497+0000 7fb2a2f4f040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:04:05.511 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setuser ceph since I am not root 2024-08-17T17:04:05.511 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setgroup ceph since I am not root 2024-08-17T17:04:05.511 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:05.359+0000 7f8e79bde040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:04:05.773 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:05.567+0000 7fb2a2f4f040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:04:05.832 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:05.510+0000 7f8e79bde040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:04:05.833 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:05.580+0000 7f8e79bde040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:04:06.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:05.818+0000 7fb2a2f4f040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:04:06.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:05.879+0000 7fb2a2f4f040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:04:06.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:05.831+0000 7f8e79bde040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:04:06.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:05.893+0000 7f8e79bde040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:04:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:06 smithi175 ceph-mon[34606]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:04:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:06 smithi175 ceph-mon[34606]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:04:06.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-mon[30574]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:04:06.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-mon[30574]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:04:06.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-mon[40964]: from='mgr.14150 172.21.15.64:0/655185674' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:04:06.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-mon[40964]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:04:07.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:06.798+0000 7fb2a2f4f040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:04:07.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:06.910+0000 7fb2a2f4f040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:04:07.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:06.965+0000 7fb2a2f4f040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:04:07.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:06.854+0000 7f8e79bde040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:04:07.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:06.973+0000 7f8e79bde040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:04:07.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.031+0000 7f8e79bde040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:04:07.445 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:07.175+0000 7fb2a2f4f040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:04:07.446 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:07.235+0000 7fb2a2f4f040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:04:07.446 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:07.300+0000 7fb2a2f4f040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:04:07.446 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:07.443+0000 7fb2a2f4f040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:04:07.531 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.250+0000 7f8e79bde040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:04:07.531 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.312+0000 7f8e79bde040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:04:07.531 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.381+0000 7f8e79bde040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:04:07.531 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.530+0000 7f8e79bde040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:04:08.160 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:07.853+0000 7fb2a2f4f040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:04:08.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:07.965+0000 7f8e79bde040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:04:08.410 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.158+0000 7fb2a2f4f040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:04:08.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.223+0000 7fb2a2f4f040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:04:08.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.368+0000 7fb2a2f4f040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:04:08.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:08.289+0000 7f8e79bde040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:04:08.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:08.357+0000 7f8e79bde040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:04:08.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:08.510+0000 7f8e79bde040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:04:09.057 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.671+0000 7fb2a2f4f040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:04:09.057 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.731+0000 7fb2a2f4f040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:04:09.057 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:08 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:08.911+0000 7fb2a2f4f040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:04:09.057 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.055+0000 7fb2a2f4f040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T17:04:09.084 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:08.829+0000 7f8e79bde040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:04:09.084 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:08 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:08.892+0000 7f8e79bde040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:04:09.390 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:09.082+0000 7f8e79bde040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:04:09.390 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:09.236+0000 7f8e79bde040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T17:04:09.390 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:09.302+0000 7f8e79bde040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:04:09.390 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:09.388+0000 7f8e79bde040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:04:09.410 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.117+0000 7fb2a2f4f040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:04:09.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.198+0000 7fb2a2f4f040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:04:09.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.257+0000 7fb2a2f4f040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:04:09.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:09.449+0000 7f8e79bde040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: Standby manager daemon x restarted 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:04:10.042 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.778+0000 7fb2a2f4f040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:09.933+0000 7fb2a2f4f040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: [17/Aug/2024:17:04:09] ENGINE Bus STARTING 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: CherryPy Checker: 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: The Application mounted at '' has an empty config. 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: [17/Aug/2024:17:04:10] ENGINE Serving on http://:::9283 2024-08-17T17:04:10.043 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[36473]: [17/Aug/2024:17:04:10] ENGINE Bus STARTED 2024-08-17T17:04:10.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: Standby manager daemon x restarted 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: Standby manager daemon x started 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: Standby manager daemon x restarted 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:04:10.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:09 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2184421369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:10.000+0000 7f8e79bde040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:10.163+0000 7f8e79bde040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:10] ENGINE Bus STARTING 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: CherryPy Checker: 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: The Application mounted at '' has an empty config. 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:10.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:10] ENGINE Serving on http://:::9283 2024-08-17T17:04:10.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:10] ENGINE Bus STARTED 2024-08-17T17:04:11.193 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled grafana update... 2024-08-17T17:04:11.258 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: Active manager daemon y restarted 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: Activating manager daemon y 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: mgrmap e18: y(active, starting, since 0.00712349s), standbys: x 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:04:11.259 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: Manager daemon y is now available 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:11.260 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:11.261 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:04:11.261 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: Active manager daemon y restarted 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: Activating manager daemon y 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: mgrmap e18: y(active, starting, since 0.00712349s), standbys: x 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:04:11.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: Manager daemon y is now available 2024-08-17T17:04:11.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: Active manager daemon y restarted 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: Activating manager daemon y 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: mgrmap e18: y(active, starting, since 0.00712349s), standbys: x 2024-08-17T17:04:11.276 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:04:11.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: Manager daemon y is now available 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:04:11.278 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:04:11.835 DEBUG:teuthology.orchestra.run.smithi175:grafana.a> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service 2024-08-17T17:04:11.838 INFO:tasks.cephadm:Setting up client nodes... 2024-08-17T17:04:11.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T17:04:12.065 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:12.181 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:04:11] "GET /metrics HTTP/1.1" 200 23676 "" "Prometheus/2.43.0" 2024-08-17T17:04:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[30574]: mgrmap e19: y(active, since 1.01072s), standbys: x 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[30574]: Saving service grafana spec with placement smithi175=a;count:1 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[40964]: mgrmap e19: y(active, since 1.01072s), standbys: x 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[40964]: Saving service grafana spec with placement smithi175=a;count:1 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:12.212+0000 7f8dd22c1640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:04:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: Traceback (most recent call last): 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._connections.run(self.expiration_interval) 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._run(expiration_interval) 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:04:12.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: s = self.context.wrap_socket( 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: return self.sslsocket_class._create( 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self.do_handshake() 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: self._sslobj.do_handshake() 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:04:12.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:04:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:12 smithi175 ceph-mon[34606]: mgrmap e19: y(active, since 1.01072s), standbys: x 2024-08-17T17:04:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:12 smithi175 ceph-mon[34606]: Saving service grafana spec with placement smithi175=a;count:1 2024-08-17T17:04:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:12 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:12 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:12 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: [17/Aug/2024:17:04:11] ENGINE Bus STARTING 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: [17/Aug/2024:17:04:12] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: [17/Aug/2024:17:04:12] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: [17/Aug/2024:17:04:12] ENGINE Bus STARTED 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: [17/Aug/2024:17:04:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: Traceback (most recent call last): 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:04:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._run(expiration_interval) 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: s = self.context.wrap_socket( 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: return self.sslsocket_class._create( 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: self.do_handshake() 2024-08-17T17:04:13.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:04:13.413 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._sslobj.do_handshake() 2024-08-17T17:04:13.413 INFO:journalctl@ceph.mon.b.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:04:13.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:13 smithi175 ceph-mon[34606]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:04:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: [17/Aug/2024:17:04:11] ENGINE Bus STARTING 2024-08-17T17:04:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: [17/Aug/2024:17:04:12] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:04:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: [17/Aug/2024:17:04:12] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: [17/Aug/2024:17:04:12] ENGINE Bus STARTED 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: [17/Aug/2024:17:04:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:04:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[30574]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: [17/Aug/2024:17:04:11] ENGINE Bus STARTING 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: [17/Aug/2024:17:04:12] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: [17/Aug/2024:17:04:12] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: [17/Aug/2024:17:04:12] ENGINE Bus STARTED 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: [17/Aug/2024:17:04:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:04:13.442 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: self.do_handshake() 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:04:13.443 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:04:13.444 INFO:journalctl@ceph.mon.c.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:04:13.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:13.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:13 smithi064 ceph-mon[40964]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:04:14.091 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2024-08-17T17:04:14.091 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQAO2MBm2kkrBRAAjUgiSpGc0/RgZzBfT3U6kQ== 2024-08-17T17:04:14.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:14 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:04:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:04:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:04:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:14 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1401945109' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:14.921 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:04:14.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-17T17:04:14.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-17T17:04:14.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T17:04:15.163 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 4016M 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:15.286 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.287 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:15 smithi175 ceph-mon[34606]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:04:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 4016M 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[40964]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 4016M 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:15.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:15 smithi064 ceph-mon[30574]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:04:16.553 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:16 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.554 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:16 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.554 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:16 smithi175 ceph-mon[34606]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:04:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[30574]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:04:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:04:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:16 smithi064 ceph-mon[40964]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:04:17.176 INFO:teuthology.orchestra.run.smithi175.stdout:[client.1] 2024-08-17T17:04:17.177 INFO:teuthology.orchestra.run.smithi175.stdout: key = AQAR2MBmTPk9ChAAmwjy95/SL/NB5cSWKEQHPA== 2024-08-17T17:04:17.461 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:17 smithi175 ceph-mon[34606]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:17.461 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:17 smithi175 ceph-mon[34606]: from='client.? 172.21.15.175:0/626061891' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.461 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:17 smithi175 ceph-mon[34606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.461 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:17 smithi175 ceph-mon[34606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[30574]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[30574]: from='client.? 172.21.15.175:0/626061891' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[30574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[30574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[40964]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[40964]: from='client.? 172.21.15.175:0/626061891' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[40964]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:04:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:17 smithi064 ceph-mon[40964]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:04:17.797 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-08-17T17:04:17.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-17T17:04:17.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-17T17:04:17.836 INFO:tasks.cephadm:Creating RBD pool 2024-08-17T17:04:17.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create rbd 8 2024-08-17T17:04:18.501 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:19 smithi064 ceph-mon[30574]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-08-17T17:04:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:19 smithi064 ceph-mon[40964]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-08-17T17:04:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:19 smithi175 ceph-mon[34606]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-08-17T17:04:21.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:21 smithi064 ceph-mon[30574]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-08-17T17:04:21.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:21 smithi064 ceph-mon[40964]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-08-17T17:04:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:21 smithi175 ceph-mon[34606]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-08-17T17:04:21.788 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:04:21] "GET /metrics HTTP/1.1" 200 23676 "" "Prometheus/2.43.0" 2024-08-17T17:04:23.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:23 smithi064 ceph-mon[30574]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:04:23.302 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'rbd' created 2024-08-17T17:04:23.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:23 smithi064 ceph-mon[40964]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:04:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:23 smithi175 ceph-mon[34606]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:04:24.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- rbd pool init rbd 2024-08-17T17:04:24.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2170361283' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[30574]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[30574]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:04:24.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[30574]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:04:24.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2170361283' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[40964]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[40964]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:04:24.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:24 smithi064 ceph-mon[40964]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:04:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:24 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2170361283' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:24 smithi175 ceph-mon[34606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:04:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:24 smithi175 ceph-mon[34606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:04:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:24 smithi175 ceph-mon[34606]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:04:24.929 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:25.031 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:04:24 smithi064 systemd[1]: Started Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:04:25.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-17T17:04:25.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:04:25.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:26.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:26 smithi064 ceph-mon[30574]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:04:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:26 smithi064 ceph-mon[30574]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:04:26.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:26 smithi064 ceph-mon[40964]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:04:26.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:26 smithi064 ceph-mon[40964]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:04:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:26 smithi175 ceph-mon[34606]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:04:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:26 smithi175 ceph-mon[34606]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:04:26.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:04:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[69258]: ts=2024-08-17T17:04:26.706Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000291604s 2024-08-17T17:04:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:27 smithi064 ceph-mon[30574]: pgmap v14: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:27 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:04:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:27 smithi064 ceph-mon[40964]: pgmap v14: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:27 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:04:27.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:27 smithi175 ceph-mon[34606]: pgmap v14: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:27 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:04:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:28 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:04:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:28 smithi175 ceph-mon[34606]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:04:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:28 smithi175 ceph-mon[34606]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:04:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[30574]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:04:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[30574]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1758922319' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:04:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[40964]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:04:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:28 smithi064 ceph-mon[40964]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:28.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:28 smithi064 bash[69586]: Writing manifest to image destination 2024-08-17T17:04:29.089 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:28 smithi064 podman[69586]: 2024-08-17 17:04:28.77262971 +0000 UTC m=+2.263192234 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T17:04:29.089 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:28 smithi064 podman[69586]: 2024-08-17 17:04:28.862332733 +0000 UTC m=+2.352895242 container create 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:29.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:04:29.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: Cluster is now healthy 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[30574]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: Cluster is now healthy 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:04:29.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-mon[40964]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:04:29.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 podman[69586]: 2024-08-17 17:04:29.089191963 +0000 UTC m=+2.579754492 container init 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:29.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 podman[69586]: 2024-08-17 17:04:29.09238412 +0000 UTC m=+2.582946629 container start 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.096Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.096Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.097Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.097Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.097Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.097Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:04:29.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:04:29.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:04:29.369 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:04:29.370 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:04:29.371 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[69655]: ts=2024-08-17T17:04:29.098Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 bash[69586]: 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 2024-08-17T17:04:29.372 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:04:29 smithi064 systemd[1]: Started Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:04:29.600 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: Cluster is now healthy 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:29.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:29 smithi175 ceph-mon[34606]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:04:30.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph config log 1 --format=json 2024-08-17T17:04:30.270 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:30.328 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:30 smithi175 bash[63471]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-17T17:04:30.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:30 smithi064 ceph-mon[30574]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:04:30.623 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:30 smithi064 ceph-mon[30574]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:30.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:30 smithi064 ceph-mon[40964]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:04:30.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:30 smithi064 ceph-mon[40964]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:30 smithi175 ceph-mon[34606]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:04:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:30 smithi175 ceph-mon[34606]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:30.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:30 smithi175 bash[63471]: Getting image source signatures 2024-08-17T17:04:30.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:30 smithi175 bash[63471]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-17T17:04:30.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:30 smithi175 bash[63471]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-17T17:04:30.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:30 smithi175 bash[63471]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-17T17:04:32.114 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:31 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:04:31] "GET /metrics HTTP/1.1" 200 28554 "" "Prometheus/2.43.0" 2024-08-17T17:04:32.132 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:04:32.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:31 smithi175 bash[63471]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-17T17:04:32.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 bash[63471]: Writing manifest to image destination 2024-08-17T17:04:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:32 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3046782705' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:04:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:32 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3046782705' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:04:32.521 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3046782705' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:04:32.521 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 podman[63471]: 2024-08-17 17:04:32.298203885 +0000 UTC m=+2.232569047 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T17:04:32.521 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 podman[63471]: 2024-08-17 17:04:32.411355171 +0000 UTC m=+2.345720358 container create 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:04:32.778 INFO:teuthology.orchestra.run.smithi064.stdout:[{"version":22,"timestamp":"2024-08-17T17:04:13.719971+0000","name":"","changes":[{"name":"osd/host:smithi175/osd_memory_target","previous_value":"4479547596","new_value":"4211112140"}]}] 2024-08-17T17:04:32.778 INFO:tasks.ceph_manager:config epoch is 22 2024-08-17T17:04:32.778 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:04:32.779 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:04:32.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:04:32.911 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 podman[63471]: 2024-08-17 17:04:32.638530619 +0000 UTC m=+2.572895771 container init 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:04:32.911 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 podman[63471]: 2024-08-17 17:04:32.642107387 +0000 UTC m=+2.576472541 container start 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:04:32.911 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.647Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.647Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.647Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.647Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:04:32.912 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:04:32.913 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:04:32.914 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:04:32.915 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.648Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.649Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[63521]: ts=2024-08-17T17:04:32.649Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 bash[63471]: 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 2024-08-17T17:04:32.916 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:04:32 smithi175 systemd[1]: Started Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:04:33.005 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:33.127 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:32 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:32] ENGINE Bus STOPPING 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:33.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:33] ENGINE Bus STOPPED 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:33] ENGINE Bus STARTING 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:33] ENGINE Serving on http://:::9283 2024-08-17T17:04:33.440 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:04:33] ENGINE Bus STARTED 2024-08-17T17:04:34.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[30574]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T17:04:34.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:34.345 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[30574]: Deploying daemon grafana.a on smithi175 2024-08-17T17:04:34.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[40964]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T17:04:34.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:34.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-mon[40964]: Deploying daemon grafana.a on smithi175 2024-08-17T17:04:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:34 smithi175 ceph-mon[34606]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T17:04:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:34 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:04:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:34 smithi175 ceph-mon[34606]: Deploying daemon grafana.a on smithi175 2024-08-17T17:04:34.809 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:04:35.187 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:04:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[69258]: ts=2024-08-17T17:04:34.708Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002709178s 2024-08-17T17:04:35.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[30574]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 297 B/s wr, 0 op/s 2024-08-17T17:04:35.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3722786561' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:04:35.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:35.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[40964]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 297 B/s wr, 0 op/s 2024-08-17T17:04:35.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3722786561' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:04:35.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:35.510 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":21,"flags":0,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":3625104862},{"type":"v1","addr":"172.21.15.64:6801","nonce":3625104862}]},"active_addr":"172.21.15.64:6801/3625104862","active_change":"2024-08-17T17:04:10.166033+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":4139680086}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1135961297}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3398196717}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1031286826}]}]} 2024-08-17T17:04:35.516 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:04:35.517 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:04:35.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:04:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:35 smithi175 ceph-mon[34606]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 297 B/s wr, 0 op/s 2024-08-17T17:04:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:35 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3722786561' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:04:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:36.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:36 smithi064 ceph-mon[30574]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:04:36.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:36 smithi064 ceph-mon[30574]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 260 B/s wr, 0 op/s 2024-08-17T17:04:36.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:36 smithi064 ceph-mon[40964]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:04:36.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:36 smithi064 ceph-mon[40964]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 260 B/s wr, 0 op/s 2024-08-17T17:04:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:36 smithi175 ceph-mon[34606]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:04:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:36 smithi175 ceph-mon[34606]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 260 B/s wr, 0 op/s 2024-08-17T17:04:37.533 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:04:37.533 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":67,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:04:35.726367+0000","last_up_change":"2024-08-17T17:03:43.486679+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6803","nonce":1608794379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6805","nonce":1608794379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6809","nonce":1608794379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6807","nonce":1608794379}]},"public_addr":"172.21.15.64:6803/1608794379","cluster_addr":"172.21.15.64:6805/1608794379","heartbeat_back_addr":"172.21.15.64:6809/1608794379","heartbeat_front_addr":"172.21.15.64:6807/1608794379","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6811","nonce":2757074067}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6813","nonce":2757074067}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6817","nonce":2757074067}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6815","nonce":2757074067}]},"public_addr":"172.21.15.64:6811/2757074067","cluster_addr":"172.21.15.64:6813/2757074067","heartbeat_back_addr":"172.21.15.64:6817/2757074067","heartbeat_front_addr":"172.21.15.64:6815/2757074067","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6819","nonce":1249488719}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6821","nonce":1249488719}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6825","nonce":1249488719}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6823","nonce":1249488719}]},"public_addr":"172.21.15.64:6819/1249488719","cluster_addr":"172.21.15.64:6821/1249488719","heartbeat_back_addr":"172.21.15.64:6825/1249488719","heartbeat_front_addr":"172.21.15.64:6823/1249488719","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6827","nonce":495130726}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6829","nonce":495130726}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6833","nonce":495130726}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6831","nonce":495130726}]},"public_addr":"172.21.15.64:6827/495130726","cluster_addr":"172.21.15.64:6829/495130726","heartbeat_back_addr":"172.21.15.64:6833/495130726","heartbeat_front_addr":"172.21.15.64:6831/495130726","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:04:37.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:37 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1403630467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:37.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:37 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1403630467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:38.155 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:04:38.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:04:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:37 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1403630467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:38.370 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:38.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:38 smithi064 ceph-mon[30574]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 230 B/s wr, 0 op/s 2024-08-17T17:04:38.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:38 smithi064 ceph-mon[40964]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 230 B/s wr, 0 op/s 2024-08-17T17:04:39.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:38 smithi175 ceph-mon[34606]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 230 B/s wr, 0 op/s 2024-08-17T17:04:40.189 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:04:40.189 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":67,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:04:35.726367+0000","last_up_change":"2024-08-17T17:03:43.486679+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6803","nonce":1608794379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6805","nonce":1608794379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6809","nonce":1608794379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6807","nonce":1608794379}]},"public_addr":"172.21.15.64:6803/1608794379","cluster_addr":"172.21.15.64:6805/1608794379","heartbeat_back_addr":"172.21.15.64:6809/1608794379","heartbeat_front_addr":"172.21.15.64:6807/1608794379","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6811","nonce":2757074067}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6813","nonce":2757074067}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6817","nonce":2757074067}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6815","nonce":2757074067}]},"public_addr":"172.21.15.64:6811/2757074067","cluster_addr":"172.21.15.64:6813/2757074067","heartbeat_back_addr":"172.21.15.64:6817/2757074067","heartbeat_front_addr":"172.21.15.64:6815/2757074067","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6819","nonce":1249488719}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6821","nonce":1249488719}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6825","nonce":1249488719}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6823","nonce":1249488719}]},"public_addr":"172.21.15.64:6819/1249488719","cluster_addr":"172.21.15.64:6821/1249488719","heartbeat_back_addr":"172.21.15.64:6825/1249488719","heartbeat_front_addr":"172.21.15.64:6823/1249488719","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6827","nonce":495130726}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6829","nonce":495130726}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6833","nonce":495130726}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6831","nonce":495130726}]},"public_addr":"172.21.15.64:6827/495130726","cluster_addr":"172.21.15.64:6829/495130726","heartbeat_back_addr":"172.21.15.64:6833/495130726","heartbeat_front_addr":"172.21.15.64:6831/495130726","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:04:40.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/67898953' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:40.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:40.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:40.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/67898953' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:40.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:40.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:40 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/67898953' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:04:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:40.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:04:40.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:04:40.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:04:40.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:04:40.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:04:40.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:04:40.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:04:40.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:04:41.360 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.363 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.363 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.370 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:41.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:41 smithi064 ceph-mon[30574]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T17:04:41.501 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:41 smithi064 ceph-mon[40964]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T17:04:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:41 smithi175 ceph-mon[34606]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T17:04:41.784 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:04:41] "GET /metrics HTTP/1.1" 200 28579 "" "Prometheus/2.43.0" 2024-08-17T17:04:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:43 smithi064 ceph-mon[30574]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:43 smithi064 ceph-mon[40964]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:43.499 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:43 smithi175 ceph-mon[34606]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-mon[34606]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:45.411 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 podman[63856]: 2024-08-17 17:04:45.364804649 +0000 UTC m=+0.118335203 container create 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, version=8.5, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, distribution-scope=public, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, name=ubi8, release=239.1651231664, io.openshift.expose-services=, maintainer=Paul Cuzner , description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8) 2024-08-17T17:04:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:45 smithi064 ceph-mon[30574]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:45.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:45 smithi064 ceph-mon[40964]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:45.737 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 podman[63856]: 2024-08-17 17:04:45.575509196 +0000 UTC m=+0.329039724 container init 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, architecture=x86_64, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-08-17T17:04:45.737 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 podman[63856]: 2024-08-17 17:04:45.578263127 +0000 UTC m=+0.331793640 container start 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vcs-type=git, version=8.5, com.redhat.component=ubi8-container, name=ubi8, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=) 2024-08-17T17:04:45.737 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 bash[63856]: 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 2024-08-17T17:04:45.737 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 systemd[1]: Started Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:04:45.987 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736480793Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736688372Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736756435Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736798898Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736810696Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736838408Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736849316Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736860034Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736871247Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736882044Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736892898Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T17:04:45.988 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736904298Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736915458Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=settings t=2024-08-17T17:04:45.736925359Z level=info msg="App mode production" 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore t=2024-08-17T17:04:45.736997596Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore t=2024-08-17T17:04:45.737023552Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.737964393Z level=info msg="Starting DB migrations" 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.738060748Z level=info msg="Executing migration" id="create migration_log table" 2024-08-17T17:04:45.989 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.738604592Z level=info msg="Executing migration" id="create user table" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.739041303Z level=info msg="Executing migration" id="add unique index user.login" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.739638121Z level=info msg="Executing migration" id="add unique index user.email" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.740085489Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.740522996Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.740889328Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.741583157Z level=info msg="Executing migration" id="create user table v2" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.741992211Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.742370196Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.742791104Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-08-17T17:04:45.990 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.743139436Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.74346401Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.743844926Z level=info msg="Executing migration" id="Update user table charset" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.744141933Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.744521764Z level=info msg="Executing migration" id="Add missing user data" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.744857595Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.746172276Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.746816855Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.747233627Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.748326051Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.748765307Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.749204929Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.749602953Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-08-17T17:04:45.991 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.750019635Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.750408723Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.7506684Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.751052289Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.751419616Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.75175301Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.752145867Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.752736786Z level=info msg="Executing migration" id="create temp_user v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.753126669Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.753507589Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.753867384Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.754246696Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.754631472Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-08-17T17:04:45.992 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.754967853Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.755300308Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.755591074Z level=info msg="Executing migration" id="create star table" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.755913944Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.756313213Z level=info msg="Executing migration" id="create org table v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.756694475Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.757095873Z level=info msg="Executing migration" id="create org_user table v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.7574543Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.757836913Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.75823921Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.758607316Z level=info msg="Executing migration" id="Update org table charset" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.758867902Z level=info msg="Executing migration" id="Update org_user table charset" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.759148423Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-08-17T17:04:45.993 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.759424851Z level=info msg="Executing migration" id="create dashboard table" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.759789224Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.760194719Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.760600482Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.761050125Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.761444832Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.761789059Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.762610471Z level=info msg="Executing migration" id="create dashboard v2" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.763013443Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-08-17T17:04:45.994 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.763410548Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-08-17T17:04:45.995 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.763816227Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-08-17T17:04:45.995 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.764142587Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-08-17T17:04:45.995 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.764493986Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-08-17T17:04:45.995 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.764746645Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-08-17T17:04:45.995 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.765222242Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.765682551Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.766203976Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.766603092Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.767086782Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.767483762Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.767847499Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.768126166Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.76837524Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.768865374Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.769357199Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.769852899Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.77037208Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-08-17T17:04:45.996 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.77270701Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.773184905Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.775695458Z level=info msg="Executing migration" id="Update dashboard title length" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.776005733Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.778891372Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.781678111Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.784398862Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.784880291Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.787798289Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.790516116Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.793254177Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.793768454Z level=info msg="Executing migration" id="Add check_sum column" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.797021539Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.799662339Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-08-17T17:04:45.997 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.802484444Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.805041873Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.807753314Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.810396883Z level=info msg="Executing migration" id="create data_source table" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.813018827Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.813470539Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.813888837Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.816610296Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.819350159Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.820410684Z level=info msg="Executing migration" id="create data_source table v2" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.823120291Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.825811551Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.828533174Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-08-17T17:04:45.998 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.828915037Z level=info msg="Executing migration" id="Add column with_credentials" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.831696792Z level=info msg="Executing migration" id="Add secure json data column" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.834446429Z level=info msg="Executing migration" id="Update data_source table charset" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.83705193Z level=info msg="Executing migration" id="Update initial version to 1" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.837400115Z level=info msg="Executing migration" id="Add read_only data column" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.839905647Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.842629412Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.845233596Z level=info msg="Executing migration" id="Add uid column" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.845797428Z level=info msg="Executing migration" id="Update uid value" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.84844555Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.849248463Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.850067256Z level=info msg="Executing migration" id="create api_key table" 2024-08-17T17:04:45.999 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.850492468Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.851319573Z level=info msg="Executing migration" id="add index api_key.key" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.852100611Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.852887376Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.853311367Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.854137934Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.854554705Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.855604313Z level=info msg="Executing migration" id="create api_key table v2" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.856392372Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.857185373Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.857971579Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.858823846Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.859611006Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.860456653Z level=info msg="Executing migration" id="Update api_key table charset" 2024-08-17T17:04:46.000 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.861342412Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.862164475Z level=info msg="Executing migration" id="Add service account foreign key" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.863004517Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.863844034Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.864699755Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.865558257Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.866466963Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.867320235Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.868105752Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.869087039Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.869843766Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.870775278Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-08-17T17:04:46.001 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.871512043Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.872339181Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.872944592Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.873499369Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.874348715Z level=info msg="Executing migration" id="create quota table v1" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.874944263Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.87540344Z level=info msg="Executing migration" id="Update quota table charset" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.87820297Z level=info msg="Executing migration" id="create plugin_setting table" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.87863019Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.880992534Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.884164241Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.887017743Z level=info msg="Executing migration" id="create session table" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.889540311Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-08-17T17:04:46.002 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.892284835Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.89512538Z level=info msg="Executing migration" id="create playlist table v2" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.897940125Z level=info msg="Executing migration" id="create playlist item table v2" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.900736613Z level=info msg="Executing migration" id="Update playlist table charset" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.903469024Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.906145112Z level=info msg="Executing migration" id="drop preferences table v2" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.90895209Z level=info msg="Executing migration" id="drop preferences table v3" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.911879483Z level=info msg="Executing migration" id="create preferences table v3" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.914601153Z level=info msg="Executing migration" id="Update preferences table charset" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.917520142Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.92029509Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.92312526Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.925860953Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-17T17:04:46.003 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.928514342Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.931216924Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.933887712Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.936498108Z level=info msg="Executing migration" id="create alert table v1" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.93915446Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.941737549Z level=info msg="Executing migration" id="add index alert state" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.94453803Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-08-17T17:04:46.004 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.947164089Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.949797844Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.952439133Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.955136995Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.957974067Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.960640155Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.963393597Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.96614605Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.9688957Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.969617319Z level=info msg="Executing migration" id="Add column is_default" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.970463212Z level=info msg="Executing migration" id="Add column frequency" 2024-08-17T17:04:46.005 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.971251347Z level=info msg="Executing migration" id="Add column send_reminder" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.972037503Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.972838348Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.973651996Z level=info msg="Executing migration" id="Update alert table charset" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.97440182Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.974710576Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.975331826Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.976113125Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.976942098Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.977718427Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.978522839Z level=info msg="Executing migration" id="Add for to alert table" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.979364278Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.980149934Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-08-17T17:04:46.006 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.980914294Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.981725804Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.982571761Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.983481914Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.984339026Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.98521694Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.986059967Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.986915377Z level=info msg="Executing migration" id="create annotation table v5" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.987800783Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.988619403Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.989480652Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.990318187Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.991184336Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-08-17T17:04:46.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.992027491Z level=info msg="Executing migration" id="Update annotation table charset" 2024-08-17T17:04:46.008 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.992867875Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-08-17T17:04:46.008 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.993712668Z level=info msg="Executing migration" id="Drop category_id index" 2024-08-17T17:04:46.008 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.994546456Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-08-17T17:04:46.008 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.995401117Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-08-17T17:04:46.008 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.996263755Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.997115858Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:45.999794963Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.002601402Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.005188368Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.007991642Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-08-17T17:04:46.238 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.010662966Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.013382545Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.016018166Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.01887828Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.021710163Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.024500312Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.027385232Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.030024302Z level=info msg="Executing migration" id="Add epoch_end column" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.032533352Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.035522125Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.038174635Z level=info msg="Executing migration" id="Move region to single row" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.040921337Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-08-17T17:04:46.239 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.043717238Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.046560559Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.047150921Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.047657361Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.048089911Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.048505421Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.048795166Z level=info msg="Executing migration" id="create test_data table" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.049239565Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.049625823Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.050058474Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.050471534Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.050846957Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.051276011Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-08-17T17:04:46.240 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.051570335Z level=info msg="Executing migration" id="create team table" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.051980658Z level=info msg="Executing migration" id="add index team.org_id" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.052545912Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.053259768Z level=info msg="Executing migration" id="create team member table" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.053637887Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.054059979Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-08-17T17:04:46.241 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.054476059Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-08-17T17:04:46.242 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.05488034Z level=info msg="Executing migration" id="Add column email to team table" 2024-08-17T17:04:46.242 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.055584319Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-08-17T17:04:46.242 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.056278006Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.056988217Z level=info msg="Executing migration" id="create dashboard acl table" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.05737431Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.057822395Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.060337125Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.060811009Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.063605138Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.066337404Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.067165386Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.068015806Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.068865943Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.069672583Z level=info msg="Executing migration" id="create tag table" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.070531521Z level=info msg="Executing migration" id="add index tag.key_value" 2024-08-17T17:04:46.243 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.071417047Z level=info msg="Executing migration" id="create login attempt table" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.071871972Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.072698426Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.07352397Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.075500197Z level=info msg="Executing migration" id="create login_attempt v2" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.075907251Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.076754753Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.077598221Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.078452024Z level=info msg="Executing migration" id="create user auth table" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.0789509Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.079768397Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.080596154Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.08144965Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-08-17T17:04:46.244 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.082280535Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.083244606Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.084208487Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.08508539Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.086229712Z level=info msg="Executing migration" id="create server_lock table" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.087081563Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.087935621Z level=info msg="Executing migration" id="create user auth token table" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.08883315Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.089458968Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.090320268Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.091163312Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.092272375Z level=info msg="Executing migration" id="create cache_data table" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.092718168Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-08-17T17:04:46.245 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.093553868Z level=info msg="Executing migration" id="create short_url table v1" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.094433786Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.095279562Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.095634876Z level=info msg="Executing migration" id="delete alert_definition table" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.098268898Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.098798833Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.101238576Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.104036321Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.106949794Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.10985603Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.11265002Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.113291217Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-08-17T17:04:46.246 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.11379135Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.114769189Z level=info msg="Executing migration" id="drop alert_definition table" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.115189858Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.115475104Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.115924265Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.116358091Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.116767933Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.117077658Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.117463505Z level=info msg="Executing migration" id="create alert_instance table" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.117857064Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.118283822Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.118715486Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-08-17T17:04:46.247 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.119528617Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.11999108Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.120406179Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.123916557Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.127561925Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.128080959Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.128689708Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.129924953Z level=info msg="Executing migration" id="create alert_rule table" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.130486653Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.131306333Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.131750464Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.132172952Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-08-17T17:04:46.248 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.132777389Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.133649559Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.134508988Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.135336855Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.135763396Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.136200916Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.137014333Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.13781437Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.138264965Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.139360939Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-17T17:04:46.249 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.140209255Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.140586908Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.141077101Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.141509485Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.141978321Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.142281456Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.143157386Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.146052821Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.147447058Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.148738185Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.149922917Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.150229054Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-08-17T17:04:46.250 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.150618046Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.151462713Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.15176188Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.152648598Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.155441967Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.156466147Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.156892701Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.15735924Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.158221715Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.158632328Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.159088171Z level=info msg="Executing migration" id="create alert_image table" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.15948997Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.160032231Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-17T17:04:46.251 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.160352191Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.160760609Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.161208035Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.161314883Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.164287444Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.167145544Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.16982326Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.170020257Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.170190976Z level=info msg="alerts found to migrate" alerts=0 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.170745561Z level=info msg="Executing migration" id="create library_element table v1" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.173500183Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-08-17T17:04:46.252 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.17624686Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.179014687Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.181773885Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.184455419Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.187191388Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.190007797Z level=info msg="Executing migration" id="create data_keys table" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.192461578Z level=info msg="Executing migration" id="create secrets table" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.195195239Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.199799236Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.202477147Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.205139727Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.209425857Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.213714017Z level=info msg="Executing migration" id="create kv_store table v1" 2024-08-17T17:04:46.253 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.216474112Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.219103414Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.221869939Z level=info msg="Executing migration" id="create permission table" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.224535205Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.227010309Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.22988712Z level=info msg="Executing migration" id="create role table" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.230553617Z level=info msg="Executing migration" id="add column display_name" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.231501124Z level=info msg="Executing migration" id="add column group_name" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.232453487Z level=info msg="Executing migration" id="add index role.org_id" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.233313873Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.234205495Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.234961856Z level=info msg="Executing migration" id="create team role table" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.235887234Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-08-17T17:04:46.254 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.236596722Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.237425337Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.238213777Z level=info msg="Executing migration" id="create user role table" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.239012876Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.239811504Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.240596672Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-08-17T17:04:46.255 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.241454397Z level=info msg="Executing migration" id="create builtin role table" 2024-08-17T17:04:46.508 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.24223994Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-08-17T17:04:46.508 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.24308677Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-08-17T17:04:46.508 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.244602269Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.245743919Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.246632921Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.247480048Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.248315128Z level=info msg="Executing migration" id="add unique index role.uid" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.249200123Z level=info msg="Executing migration" id="create seed assignment table" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.250147128Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.251129074Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.252202455Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.253085202Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.254028895Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.254775841Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-17T17:04:46.509 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.255703338Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-17T17:04:46.510 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.256445917Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-17T17:04:46.510 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.257254983Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-17T17:04:46.510 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.257560018Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-17T17:04:46.510 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.258111104Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.258500159Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.258997209Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.261740152Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.264450161Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.267234155Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.270077112Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-17T17:04:46.511 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.272712491Z level=info msg="Executing migration" id="add correlation config column" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.275406562Z level=info msg="Executing migration" id="create entity_events table" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.278126653Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.280981025Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.281064673Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.283802296Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.283909038Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.286582741Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.289436198Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.292101902Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.294751094Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.29761614Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.300221591Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T17:04:46.512 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.302959095Z level=info msg="Executing migration" id="Drop public config table" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.305796173Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.308430337Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.311173284Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.313934654Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.316721547Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.31975773Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.322336238Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.325014995Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.327746295Z level=info msg="Executing migration" id="add share column" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.33031323Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.332845267Z level=info msg="Executing migration" id="create file table" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.335492008Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.33824226Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-17T17:04:46.513 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.341074439Z level=info msg="Executing migration" id="create file_meta table" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.343793378Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.346306818Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.34904218Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.351736Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.354444665Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.355252263Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.356270159Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.357087095Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.357904049Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.358695763Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-17T17:04:46.514 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.359568042Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.360258282Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.361314782Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.36234161Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.363181808Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.373114875Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.373930731Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-17T17:04:46.515 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.374750501Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.378079562Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.378849161Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.379783597Z level=info msg="Executing migration" id="create folder table" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.380640281Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.381331013Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.3823102Z level=info msg="Executing migration" id="Update folder title length" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.383060522Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=migrator t=2024-08-17T17:04:46.383921238Z level=info msg="migrations completed" performed=485 skipped=0 duration=645.888081ms 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore t=2024-08-17T17:04:46.384308921Z level=info msg="Created default organization" 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=plugin.loader t=2024-08-17T17:04:46.439816617Z level=info msg="Plugin registered" pluginID=input 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=plugin.loader t=2024-08-17T17:04:46.46820578Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T17:04:46.516 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=plugin.loader t=2024-08-17T17:04:46.468233848Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T17:04:46.517 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=secrets t=2024-08-17T17:04:46.468271368Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T17:04:46.517 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=query_data t=2024-08-17T17:04:46.469993018Z level=info msg="Query Service initialization" 2024-08-17T17:04:46.517 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=live.push_http t=2024-08-17T17:04:46.47238975Z level=info msg="Live Push Gateway initialization" 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:46 smithi175 ceph-mon[34606]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:47.055 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.055 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.055 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.055 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.055 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[30574]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:04:47.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:46 smithi064 ceph-mon[40964]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:47.493 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=infra.usagestats.collector t=2024-08-17T17:04:47.223385001Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T17:04:47.493 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=provisioning.datasources t=2024-08-17T17:04:47.279479571Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=provisioning.datasources t=2024-08-17T17:04:47.311205905Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=provisioning.alerting t=2024-08-17T17:04:47.333922291Z level=info msg="starting to provision alerting" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=provisioning.alerting t=2024-08-17T17:04:47.333941809Z level=info msg="finished to provision alerting" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=ngalert.state.manager t=2024-08-17T17:04:47.334008825Z level=info msg="Warming state cache for startup" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=grafanaStorageLogger t=2024-08-17T17:04:47.33417494Z level=info msg="storage starting" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=http.server t=2024-08-17T17:04:47.336027959Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=ngalert.state.manager t=2024-08-17T17:04:47.344298716Z level=info msg="State cache has been initialized" states=0 duration=10.288766ms 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=ngalert.multiorg.alertmanager t=2024-08-17T17:04:47.344335903Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=ticker t=2024-08-17T17:04:47.34435432Z level=info msg=starting first_tick=2024-08-17T17:04:50Z 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore.transactions t=2024-08-17T17:04:47.344369145Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore.transactions t=2024-08-17T17:04:47.389165237Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore.transactions t=2024-08-17T17:04:47.399554308Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-08-17T17:04:47.494 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:04:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[63871]: logger=sqlstore.transactions t=2024-08-17T17:04:47.436351572Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:04:49.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:49 smithi175 ceph-mon[34606]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:49 smithi064 ceph-mon[30574]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:49.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:49 smithi064 ceph-mon[40964]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[30574]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[40964]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:50 smithi175 ceph-mon[34606]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:04:51.668 INFO:teuthology.orchestra.run.smithi064.stdout:236223201294 2024-08-17T17:04:51.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:04:51.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:04:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:04:51] "GET /metrics HTTP/1.1" 200 28579 "" "Prometheus/2.43.0" 2024-08-17T17:04:52.205 INFO:teuthology.orchestra.run.smithi064.stdout:55834574892 2024-08-17T17:04:52.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:04:53.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:53 smithi064 ceph-mon[30574]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:53.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:53 smithi064 ceph-mon[40964]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:53.553 INFO:teuthology.orchestra.run.smithi064.stdout:111669149731 2024-08-17T17:04:53.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:04:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:53 smithi175 ceph-mon[34606]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:54.514 INFO:teuthology.orchestra.run.smithi064.stdout:34359738418 2024-08-17T17:04:54.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:04:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:55 smithi064 ceph-mon[30574]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:55 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:55 smithi064 ceph-mon[40964]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:55 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:55 smithi175 ceph-mon[34606]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:55 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:04:55.897 INFO:teuthology.orchestra.run.smithi064.stdout:167503724569 2024-08-17T17:04:55.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:04:56.802 INFO:teuthology.orchestra.run.smithi064.stdout:201863462933 2024-08-17T17:04:56.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:04:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:57 smithi064 ceph-mon[30574]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:57.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:57 smithi064 ceph-mon[40964]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:57 smithi175 ceph-mon[34606]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:57.836 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:57.839 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:58.603 INFO:teuthology.orchestra.run.smithi064.stdout:77309411368 2024-08-17T17:04:58.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:04:59.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:04:59 smithi064 ceph-mon[30574]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:59.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:04:59 smithi064 ceph-mon[40964]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:59.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:59.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:59.552 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:59.553 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:04:59 smithi175 ceph-mon[34606]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:01.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:01 smithi064 ceph-mon[30574]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:01.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:01 smithi064 ceph-mon[40964]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:01.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:01 smithi175 ceph-mon[34606]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:02.049 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:05:01] "GET /metrics HTTP/1.1" 200 28574 "" "Prometheus/2.43.0" 2024-08-17T17:05:03.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:03 smithi064 ceph-mon[30574]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:03.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:03 smithi064 ceph-mon[40964]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:03 smithi175 ceph-mon[34606]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:04.286 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:05.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:05 smithi064 ceph-mon[30574]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:05.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:05 smithi064 ceph-mon[40964]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:05.404 INFO:teuthology.orchestra.run.smithi064.stdout:111669149733 2024-08-17T17:05:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:05 smithi175 ceph-mon[34606]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:06.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:06 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/474677949' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:05:06.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:06 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/474677949' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:05:06.462 INFO:teuthology.orchestra.run.smithi064.stdout:34359738421 2024-08-17T17:05:06.653 INFO:teuthology.orchestra.run.smithi064.stdout:236223201297 2024-08-17T17:05:06.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:06 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/474677949' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:05:07.046 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738418 got 34359738421 for osd.0 2024-08-17T17:05:07.046 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:07.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[30574]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:07.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2525789104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:05:07.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1884910109' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:05:07.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[40964]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:07.326 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2525789104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:05:07.326 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:07 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1884910109' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:05:07.500 INFO:teuthology.orchestra.run.smithi064.stdout:167503724572 2024-08-17T17:05:07.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:07 smithi175 ceph-mon[34606]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:07 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2525789104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:05:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:07 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1884910109' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:05:08.062 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724572 for osd.5 2024-08-17T17:05:08.062 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:08.434 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:08 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2566392219' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:05:08.434 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:08 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2566392219' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:05:08.457 INFO:teuthology.orchestra.run.smithi064.stdout:133143986207 2024-08-17T17:05:08.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:05:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:08 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2566392219' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:05:09.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:09 smithi064 ceph-mon[30574]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:09.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:09 smithi064 ceph-mon[40964]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:09.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:09 smithi175 ceph-mon[34606]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:10.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:10.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:10.865 INFO:teuthology.orchestra.run.smithi064.stdout:55834574896 2024-08-17T17:05:11.148 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149731 got 111669149733 for osd.3 2024-08-17T17:05:11.148 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:11.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:11 smithi064 ceph-mon[40964]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:11.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:11 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/4041728290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:05:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:11 smithi064 ceph-mon[30574]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:11 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/4041728290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:05:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:11 smithi175 ceph-mon[34606]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:11.699 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:11 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/4041728290' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:05:11.847 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:05:11] "GET /metrics HTTP/1.1" 200 28578 "" "Prometheus/2.43.0" 2024-08-17T17:05:12.036 INFO:teuthology.orchestra.run.smithi064.stdout:77309411370 2024-08-17T17:05:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:12 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2669264963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:05:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:12 smithi064 ceph-mon[30574]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:12 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2669264963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:05:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:12 smithi064 ceph-mon[40964]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:12 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2669264963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:05:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:12 smithi175 ceph-mon[34606]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:12.699 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411370 for osd.2 2024-08-17T17:05:12.699 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:13.384 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201297 for osd.7 2024-08-17T17:05:13.384 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:13.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-08-17T17:05:13.972 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:14.330 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:15.200 INFO:teuthology.orchestra.run.smithi064.stdout:201863462937 2024-08-17T17:05:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2526410173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:15 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2526410173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:05:15.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:15 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2526410173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:05:16.055 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462933 got 201863462937 for osd.6 2024-08-17T17:05:16.055 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:16 smithi064 ceph-mon[30574]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:05:16.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:16 smithi064 ceph-mon[30574]: Reconfiguring daemon alertmanager.a on smithi064 2024-08-17T17:05:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:16 smithi064 ceph-mon[40964]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:05:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:16 smithi064 ceph-mon[40964]: Reconfiguring daemon alertmanager.a on smithi064 2024-08-17T17:05:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:16 smithi175 ceph-mon[34606]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:05:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:16 smithi175 ceph-mon[34606]: Reconfiguring daemon alertmanager.a on smithi064 2024-08-17T17:05:17.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:17 smithi175 ceph-mon[34606]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:17 smithi064 ceph-mon[30574]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:17 smithi064 ceph-mon[40964]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:18.512 INFO:teuthology.orchestra.run.smithi064.stdout:133143986210 2024-08-17T17:05:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:19 smithi064 ceph-mon[30574]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:19 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3261272561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:05:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:19 smithi064 ceph-mon[40964]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3261272561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:05:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:19 smithi175 ceph-mon[34606]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:19 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3261272561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:05:20.060 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:19 smithi064 systemd[1]: Stopping Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:05:20.285 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986207 got 133143986210 for osd.4 2024-08-17T17:05:20.285 DEBUG:teuthology.parallel:result is None 2024-08-17T17:05:20.285 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:05:20.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:05:20.687 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[69258]: ts=2024-08-17T17:05:20.537Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:05:20.687 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:20 smithi064 podman[73608]: 2024-08-17 17:05:20.558007357 +0000 UTC m=+0.108001421 container died 3c395d9870a9569e7dfcc388bb4fade05c6d62ba29d0b0aeb374b6129632bcf1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:21.014 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:21.187 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:20 smithi064 podman[73608]: 2024-08-17 17:05:20.928755821 +0000 UTC m=+0.478749886 container cleanup 3c395d9870a9569e7dfcc388bb4fade05c6d62ba29d0b0aeb374b6129632bcf1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:21.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:20 smithi064 bash[73608]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:05:21.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:21 smithi064 ceph-mon[30574]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:21.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:21 smithi064 ceph-mon[40964]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:21.466 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:21 smithi064 podman[73640]: 2024-08-17 17:05:21.213168449 +0000 UTC m=+0.654147783 container remove 3c395d9870a9569e7dfcc388bb4fade05c6d62ba29d0b0aeb374b6129632bcf1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:21.466 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:21 smithi064 podman[73640]: 2024-08-17 17:05:21.288916223 +0000 UTC m=+0.729895555 volume remove 4191aa72a914909919a0d18b59482a313fd61ee0017033647113e61e6abbf76e 2024-08-17T17:05:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:21 smithi175 ceph-mon[34606]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:21.715 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:05:21] "GET /metrics HTTP/1.1" 200 28578 "" "Prometheus/2.43.0" 2024-08-17T17:05:23.158 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:23.159 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:05:23.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:22 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:05:23.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:22 smithi064 systemd[1]: Stopped Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:05:23.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:22 smithi064 systemd[1]: Starting Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:05:23.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-mon[30574]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:23.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-mon[40964]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:23.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 podman[73825]: 2024-08-17 17:05:23.105193134 +0000 UTC m=+0.018138932 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T17:05:23.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 podman[73825]: 2024-08-17 17:05:23.241684609 +0000 UTC m=+0.154630396 volume create b9eed76b9cd5e34d5ede235aad978202f433bd1a8089223efba2a35ceb568ea0 2024-08-17T17:05:23.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 podman[73825]: 2024-08-17 17:05:23.333839045 +0000 UTC m=+0.246784826 container create ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:23 smithi175 ceph-mon[34606]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:23.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 podman[73825]: 2024-08-17 17:05:23.67039574 +0000 UTC m=+0.583341542 container init ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:23.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 podman[73825]: 2024-08-17 17:05:23.673529837 +0000 UTC m=+0.586475620 container start ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:23.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.686Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T17:05:23.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.686Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T17:05:23.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.687Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.688Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.713Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.713Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.716Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:23.716Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 bash[73825]: ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 2024-08-17T17:05:24.188 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:23 smithi064 systemd[1]: Started Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:05:24.380 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2024-08-17T17:05:22.190758+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227284,"kb_used_data":7028,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733932,"statfs":{"total":767960285184,"available":767727546368,"internally_reserved":0,"allocated":7196672,"data_stored":4056080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002908"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940794+0000","last_change":"2024-08-17T17:04:29.332851+0000","last_active":"2024-08-17T17:04:35.940794+0000","last_peered":"2024-08-17T17:04:35.940794+0000","last_clean":"2024-08-17T17:04:35.940794+0000","last_became_active":"2024-08-17T17:04:24.312005+0000","last_became_peered":"2024-08-17T17:04:24.312005+0000","last_unstale":"2024-08-17T17:04:35.940794+0000","last_undegraded":"2024-08-17T17:04:35.940794+0000","last_fullsized":"2024-08-17T17:04:35.940794+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:53.614800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029301899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940425+0000","last_change":"2024-08-17T17:04:29.328607+0000","last_active":"2024-08-17T17:04:35.940425+0000","last_peered":"2024-08-17T17:04:35.940425+0000","last_clean":"2024-08-17T17:04:35.940425+0000","last_became_active":"2024-08-17T17:04:24.349906+0000","last_became_peered":"2024-08-17T17:04:24.349906+0000","last_unstale":"2024-08-17T17:04:35.940425+0000","last_undegraded":"2024-08-17T17:04:35.940425+0000","last_fullsized":"2024-08-17T17:04:35.940425+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:29.809856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027426399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940700+0000","last_change":"2024-08-17T17:04:29.455481+0000","last_active":"2024-08-17T17:04:35.940700+0000","last_peered":"2024-08-17T17:04:35.940700+0000","last_clean":"2024-08-17T17:04:35.940700+0000","last_became_active":"2024-08-17T17:04:24.350074+0000","last_became_peered":"2024-08-17T17:04:24.350074+0000","last_unstale":"2024-08-17T17:04:35.940700+0000","last_undegraded":"2024-08-17T17:04:35.940700+0000","last_fullsized":"2024-08-17T17:04:35.940700+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:30.046202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025965899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940445+0000","last_change":"2024-08-17T17:04:29.328629+0000","last_active":"2024-08-17T17:04:35.940445+0000","last_peered":"2024-08-17T17:04:35.940445+0000","last_clean":"2024-08-17T17:04:35.940445+0000","last_became_active":"2024-08-17T17:04:24.349917+0000","last_became_peered":"2024-08-17T17:04:24.349917+0000","last_unstale":"2024-08-17T17:04:35.940445+0000","last_undegraded":"2024-08-17T17:04:35.940445+0000","last_fullsized":"2024-08-17T17:04:35.940445+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:53.984346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029370500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940566+0000","last_change":"2024-08-17T17:04:29.327930+0000","last_active":"2024-08-17T17:04:35.940566+0000","last_peered":"2024-08-17T17:04:35.940566+0000","last_clean":"2024-08-17T17:04:35.940566+0000","last_became_active":"2024-08-17T17:04:24.310910+0000","last_became_peered":"2024-08-17T17:04:24.310910+0000","last_unstale":"2024-08-17T17:04:35.940566+0000","last_undegraded":"2024-08-17T17:04:35.940566+0000","last_fullsized":"2024-08-17T17:04:35.940566+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:37.573482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083398900000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.732637+0000","last_change":"2024-08-17T17:04:29.316120+0000","last_active":"2024-08-17T17:04:35.732637+0000","last_peered":"2024-08-17T17:04:35.732637+0000","last_clean":"2024-08-17T17:04:35.732637+0000","last_became_active":"2024-08-17T17:04:24.350215+0000","last_became_peered":"2024-08-17T17:04:24.350215+0000","last_unstale":"2024-08-17T17:04:35.732637+0000","last_undegraded":"2024-08-17T17:04:35.732637+0000","last_fullsized":"2024-08-17T17:04:35.732637+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:49.696633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026442399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940544+0000","last_change":"2024-08-17T17:04:29.455410+0000","last_active":"2024-08-17T17:04:35.940544+0000","last_peered":"2024-08-17T17:04:35.940544+0000","last_clean":"2024-08-17T17:04:35.940544+0000","last_became_active":"2024-08-17T17:04:24.349707+0000","last_became_peered":"2024-08-17T17:04:24.349707+0000","last_unstale":"2024-08-17T17:04:35.940544+0000","last_undegraded":"2024-08-17T17:04:35.940544+0000","last_fullsized":"2024-08-17T17:04:35.940544+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:31.934322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024844199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940562+0000","last_change":"2024-08-17T17:04:29.327959+0000","last_active":"2024-08-17T17:04:35.940562+0000","last_peered":"2024-08-17T17:04:35.940562+0000","last_clean":"2024-08-17T17:04:35.940562+0000","last_became_active":"2024-08-17T17:04:24.310628+0000","last_became_peered":"2024-08-17T17:04:24.310628+0000","last_unstale":"2024-08-17T17:04:35.940562+0000","last_undegraded":"2024-08-17T17:04:35.940562+0000","last_fullsized":"2024-08-17T17:04:35.940562+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:34.789294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083540999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:04:29.455212+0000","last_change":"2024-08-17T17:03:46.726080+0000","last_active":"2024-08-17T17:04:29.455212+0000","last_peered":"2024-08-17T17:04:29.455212+0000","last_clean":"2024-08-17T17:04:29.455212+0000","last_became_active":"2024-08-17T17:03:46.725689+0000","last_became_peered":"2024-08-17T17:03:46.725689+0000","last_unstale":"2024-08-17T17:04:29.455212+0000","last_undegraded":"2024-08-17T17:04:29.455212+0000","last_fullsized":"2024-08-17T17:04:29.455212+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:54.215266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":876011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":13.132999999999999,"5min":13.132999999999999,"15min":13.132999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.495}]},{"osd":1,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.65800000000000003}]},{"osd":2,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.505},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":19.109999999999999,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.628}]},{"osd":3,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.51400000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.435}]},{"osd":5,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.47399999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":875765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.71499999999999997}]},{"osd":1,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.63800000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.749}]},{"osd":4,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.503},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Aug 17 17:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.625},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Aug 17 17:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.69999999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.377},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52700000000000002,"15min":0.52400000000000002},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39400000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.502,"15min":0.495},"min":{"1min":0.34000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.44},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54800000000000004,"15min":0.54300000000000004},"min":{"1min":0.438,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.374,"15min":0.36799999999999999},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.53000000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.35399999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Aug 17 17:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.316},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.29399999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.26,"15min":1.3859999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69699999999999995,"5min":16.233000000000001,"15min":16.233000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.64500000000000002,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.2450000000000001,"15min":1.369},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.63700000000000001,"5min":16.245000000000001,"15min":16.245000000000001},"last":0.436},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56999999999999995,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.749,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52800000000000002,"15min":0.53100000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70499999999999996,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70899999999999996,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.52400000000000002,"5min":1.2410000000000001,"15min":1.361},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.66600000000000004,"5min":16.27,"15min":16.27},"last":0.5}]},{"osd":5,"last update":"Sat Aug 17 17:04:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57999999999999996,"5min":0.629,"15min":0.629},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.39000000000000001,"15min":0.38400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40300000000000002,"15min":0.39600000000000002},"min":{"1min":0.30299999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Aug 17 17:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.51200000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38500000000000001,"15min":0.39200000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.58999999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.40500000000000003,"15min":0.41599999999999998},"min":{"1min":0.249,"5min":0.246,"15min":0.246},"max":{"1min":0.57399999999999995,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.33500000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42599999999999999,"15min":0.44},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.51300000000000001,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40200000000000002,"15min":0.41399999999999998},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51000000000000001,"5min":0.621,"15min":0.621},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44700000000000001,"15min":0.45600000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65000000000000002,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.377,"5min":0.42999999999999999,"15min":0.439},"min":{"1min":0.30399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58799999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.624}]},{"osd":5,"last update":"Sat Aug 17 17:04:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.40799999999999997,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81200000000000006,"5min":0.877,"15min":0.877},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.75,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.59599999999999997}]},{"osd":6,"last update":"Sat Aug 17 17:04:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Aug 17 17:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54800000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.33700000000000002},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.215},{"interface":"front","average":{"1min":0.375,"5min":0.36199999999999999,"15min":0.34999999999999998},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.51400000000000001,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.315}]},{"osd":1,"last update":"Sat Aug 17 17:04:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35099999999999998,"15min":0.34000000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48099999999999998,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.377},"min":{"1min":0.311,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.61399999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.26500000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.44600000000000001,"15min":0.442},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60499999999999998,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.41899999999999998,"15min":0.42599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56299999999999994,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.24299999999999999}]},{"osd":4,"last update":"Sat Aug 17 17:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.40300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.376},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.432,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76700000000000002,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Aug 17 17:04:24 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:04:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.437},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.45400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.315},"min":{"1min":0.24099999999999999,"5min":0.155,"15min":0.155},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.34799999999999998,"15min":0.318},"min":{"1min":0.25600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.53900000000000003,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:04:52 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.39600000000000002,"15min":0.372},"min":{"1min":0.252,"5min":0.245,"15min":0.245},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.379},{"interface":"front","average":{"1min":0.438,"5min":0.39200000000000002,"15min":0.379},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.45000000000000001,"15min":0.438},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.317}]},{"osd":4,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.41499999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.59899999999999998,"15min":0.59299999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81999999999999995,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Aug 17 17:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.442},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.41499999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32360,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712792,"statfs":{"total":95995035648,"available":95961899008,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.39200000000000002,"15min":0.40899999999999997},"min":{"1min":0.20499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.48999999999999999,"5min":2.0739999999999998,"15min":2.0739999999999998},"last":0.185},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.25900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.61499999999999999,"5min":2.008,"15min":2.008},"last":0.20699999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45500000000000002,"15min":0.47299999999999998},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.55400000000000005,"5min":2.1429999999999998,"15min":2.1429999999999998},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.38600000000000001,"15min":0.376},"min":{"1min":0.33300000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.58499999999999996,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.22700000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.56899999999999995,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.255},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.443,"15min":0.443},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60099999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.26400000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.54300000000000004,"15min":0.53300000000000003},"min":{"1min":0.432,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.53600000000000003,"15min":0.53000000000000003},"min":{"1min":0.35599999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.434}]},{"osd":5,"last update":"Sat Aug 17 17:04:59 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.53800000000000003,"15min":0.52400000000000002},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.51000000000000001,"15min":0.49199999999999999},"min":{"1min":0.40100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.38100000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.37},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Aug 17 17:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.496}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:05:24.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:05:24.596 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:24.650 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:24 smithi175 ceph-mon[34606]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:24.650 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:24 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:24.650 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:24 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[30574]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:24.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[40964]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:24 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[30574]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:05:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[30574]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:05:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[30574]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[40964]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:05:25.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[40964]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:05:25.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[40964]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:25.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:25.579 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:25 smithi175 ceph-mon[34606]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:05:25.579 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:25 smithi175 ceph-mon[34606]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:05:25.579 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:25 smithi175 ceph-mon[34606]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:25.580 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:25.787 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:25.689Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001081506s 2024-08-17T17:05:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:26.311 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:05:27.114 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":48,"stamp":"2024-08-17T17:05:26.191295+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227284,"kb_used_data":7028,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733932,"statfs":{"total":767960285184,"available":767727546368,"internally_reserved":0,"allocated":7196672,"data_stored":4056080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002453"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940794+0000","last_change":"2024-08-17T17:04:29.332851+0000","last_active":"2024-08-17T17:04:35.940794+0000","last_peered":"2024-08-17T17:04:35.940794+0000","last_clean":"2024-08-17T17:04:35.940794+0000","last_became_active":"2024-08-17T17:04:24.312005+0000","last_became_peered":"2024-08-17T17:04:24.312005+0000","last_unstale":"2024-08-17T17:04:35.940794+0000","last_undegraded":"2024-08-17T17:04:35.940794+0000","last_fullsized":"2024-08-17T17:04:35.940794+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:53.614800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029301899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940425+0000","last_change":"2024-08-17T17:04:29.328607+0000","last_active":"2024-08-17T17:04:35.940425+0000","last_peered":"2024-08-17T17:04:35.940425+0000","last_clean":"2024-08-17T17:04:35.940425+0000","last_became_active":"2024-08-17T17:04:24.349906+0000","last_became_peered":"2024-08-17T17:04:24.349906+0000","last_unstale":"2024-08-17T17:04:35.940425+0000","last_undegraded":"2024-08-17T17:04:35.940425+0000","last_fullsized":"2024-08-17T17:04:35.940425+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:29.809856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027426399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940700+0000","last_change":"2024-08-17T17:04:29.455481+0000","last_active":"2024-08-17T17:04:35.940700+0000","last_peered":"2024-08-17T17:04:35.940700+0000","last_clean":"2024-08-17T17:04:35.940700+0000","last_became_active":"2024-08-17T17:04:24.350074+0000","last_became_peered":"2024-08-17T17:04:24.350074+0000","last_unstale":"2024-08-17T17:04:35.940700+0000","last_undegraded":"2024-08-17T17:04:35.940700+0000","last_fullsized":"2024-08-17T17:04:35.940700+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:30.046202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025965899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940445+0000","last_change":"2024-08-17T17:04:29.328629+0000","last_active":"2024-08-17T17:04:35.940445+0000","last_peered":"2024-08-17T17:04:35.940445+0000","last_clean":"2024-08-17T17:04:35.940445+0000","last_became_active":"2024-08-17T17:04:24.349917+0000","last_became_peered":"2024-08-17T17:04:24.349917+0000","last_unstale":"2024-08-17T17:04:35.940445+0000","last_undegraded":"2024-08-17T17:04:35.940445+0000","last_fullsized":"2024-08-17T17:04:35.940445+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:53.984346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029370500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940566+0000","last_change":"2024-08-17T17:04:29.327930+0000","last_active":"2024-08-17T17:04:35.940566+0000","last_peered":"2024-08-17T17:04:35.940566+0000","last_clean":"2024-08-17T17:04:35.940566+0000","last_became_active":"2024-08-17T17:04:24.310910+0000","last_became_peered":"2024-08-17T17:04:24.310910+0000","last_unstale":"2024-08-17T17:04:35.940566+0000","last_undegraded":"2024-08-17T17:04:35.940566+0000","last_fullsized":"2024-08-17T17:04:35.940566+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:37.573482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083398900000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.732637+0000","last_change":"2024-08-17T17:04:29.316120+0000","last_active":"2024-08-17T17:04:35.732637+0000","last_peered":"2024-08-17T17:04:35.732637+0000","last_clean":"2024-08-17T17:04:35.732637+0000","last_became_active":"2024-08-17T17:04:24.350215+0000","last_became_peered":"2024-08-17T17:04:24.350215+0000","last_unstale":"2024-08-17T17:04:35.732637+0000","last_undegraded":"2024-08-17T17:04:35.732637+0000","last_fullsized":"2024-08-17T17:04:35.732637+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:49.696633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026442399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940544+0000","last_change":"2024-08-17T17:04:29.455410+0000","last_active":"2024-08-17T17:04:35.940544+0000","last_peered":"2024-08-17T17:04:35.940544+0000","last_clean":"2024-08-17T17:04:35.940544+0000","last_became_active":"2024-08-17T17:04:24.349707+0000","last_became_peered":"2024-08-17T17:04:24.349707+0000","last_unstale":"2024-08-17T17:04:35.940544+0000","last_undegraded":"2024-08-17T17:04:35.940544+0000","last_fullsized":"2024-08-17T17:04:35.940544+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:31.934322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024844199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T17:04:35.940562+0000","last_change":"2024-08-17T17:04:29.327959+0000","last_active":"2024-08-17T17:04:35.940562+0000","last_peered":"2024-08-17T17:04:35.940562+0000","last_clean":"2024-08-17T17:04:35.940562+0000","last_became_active":"2024-08-17T17:04:24.310628+0000","last_became_peered":"2024-08-17T17:04:24.310628+0000","last_unstale":"2024-08-17T17:04:35.940562+0000","last_undegraded":"2024-08-17T17:04:35.940562+0000","last_fullsized":"2024-08-17T17:04:35.940562+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:34.789294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083540999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":263,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:04:29.455212+0000","last_change":"2024-08-17T17:03:46.726080+0000","last_active":"2024-08-17T17:04:29.455212+0000","last_peered":"2024-08-17T17:04:29.455212+0000","last_clean":"2024-08-17T17:04:29.455212+0000","last_became_active":"2024-08-17T17:03:46.725689+0000","last_became_peered":"2024-08-17T17:03:46.725689+0000","last_unstale":"2024-08-17T17:04:29.455212+0000","last_undegraded":"2024-08-17T17:04:29.455212+0000","last_fullsized":"2024-08-17T17:04:29.455212+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:54.215266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":876011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":13.132999999999999,"5min":13.132999999999999,"15min":13.132999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.47999999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":19.109999999999999,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.622},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.61199999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462940,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":875765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65600000000000003,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.5169999999999999,"5min":0.93000000000000005,"15min":0.66600000000000004},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":19.114999999999998,"5min":19.114999999999998,"15min":19.114999999999998},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.51200000000000001,"15min":0.51600000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71699999999999997,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.69999999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.496}]},{"osd":2,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52600000000000002,"15min":0.52800000000000002},"min":{"1min":0.36699999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.71199999999999997,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.67600000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.72699999999999998,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54300000000000004,"15min":0.54200000000000004},"min":{"1min":0.43099999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.52800000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.38700000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.41899999999999998,"15min":0.41099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42499999999999999,"15min":0.41599999999999998},"min":{"1min":0.309,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.46500000000000002,"15min":0.45500000000000002},"min":{"1min":0.31,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Aug 17 17:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.435}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.432},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52700000000000002,"15min":0.52400000000000002},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.495}]},{"osd":1,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39400000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.502,"15min":0.495},"min":{"1min":0.34000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.56599999999999995}]},{"osd":3,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54800000000000004,"15min":0.54300000000000004},"min":{"1min":0.438,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.626}]},{"osd":4,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.374,"15min":0.36799999999999999},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.53000000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Aug 17 17:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.314,"5min":0.313,"15min":0.313},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.68400000000000005}]},{"osd":7,"last update":"Sat Aug 17 17:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.61099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.26,"15min":1.3859999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69699999999999995,"5min":16.233000000000001,"15min":16.233000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.64500000000000002,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.628}]},{"osd":1,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.2450000000000001,"15min":1.369},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.63700000000000001,"5min":16.245000000000001,"15min":16.245000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56999999999999995,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.749,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55000000000000004}]},{"osd":2,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52800000000000002,"15min":0.53100000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70499999999999996,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.505,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70899999999999996,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.52400000000000002,"5min":1.2410000000000001,"15min":1.361},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.66600000000000004,"5min":16.27,"15min":16.27},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Aug 17 17:04:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57999999999999996,"5min":0.629,"15min":0.629},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.39000000000000001,"15min":0.38400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.435}]},{"osd":6,"last update":"Sat Aug 17 17:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.374},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40300000000000002,"15min":0.39600000000000002},"min":{"1min":0.30299999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.42199999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38500000000000001,"15min":0.39200000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.58999999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.40500000000000003,"15min":0.41599999999999998},"min":{"1min":0.249,"5min":0.246,"15min":0.246},"max":{"1min":0.57399999999999995,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.42599999999999999,"15min":0.44},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.51300000000000001,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.40200000000000002,"15min":0.41399999999999998},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51000000000000001,"5min":0.621,"15min":0.621},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:05:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44700000000000001,"15min":0.45600000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65000000000000002,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.377,"5min":0.42999999999999999,"15min":0.439},"min":{"1min":0.30399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58799999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.76500000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:04:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.40799999999999997,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81200000000000006,"5min":0.877,"15min":0.877},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.75,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.79300000000000004}]},{"osd":6,"last update":"Sat Aug 17 17:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42799999999999999,"5min":0.373,"15min":0.373},"max":{"1min":0.77400000000000002,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.48899999999999999,"5min":0.374,"15min":0.374},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.81499999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.33700000000000002},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.375},{"interface":"front","average":{"1min":0.375,"5min":0.36199999999999999,"15min":0.34999999999999998},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.51400000000000001,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.314}]},{"osd":1,"last update":"Sat Aug 17 17:04:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35099999999999998,"15min":0.34000000000000002},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48099999999999998,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.377},"min":{"1min":0.311,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.61399999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:05:18 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.44600000000000001,"15min":0.442},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60499999999999998,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.433},{"interface":"front","average":{"1min":0.38,"5min":0.41899999999999998,"15min":0.42599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56299999999999994,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:04:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.40300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.432,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Aug 17 17:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76700000000000002,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.64800000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:04:24 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Aug 17 17:04:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.54300000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574900,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.315},"min":{"1min":0.24099999999999999,"5min":0.155,"15min":0.155},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.34799999999999998,"15min":0.318},"min":{"1min":0.25600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.53900000000000003,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Aug 17 17:04:52 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.39600000000000002,"15min":0.372},"min":{"1min":0.252,"5min":0.245,"15min":0.245},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.438,"5min":0.39200000000000002,"15min":0.379},"min":{"1min":0.251,"5min":0.189,"15min":0.189},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.44700000000000001,"15min":0.437},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.44},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:04:35 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.41499999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.626},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.59899999999999998,"15min":0.59299999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81999999999999995,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Aug 17 17:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Aug 17 17:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.64800000000000002,"15min":0.66400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59099999999999997,"15min":0.59799999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Aug 17 17:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.65900000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738425,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32360,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712792,"statfs":{"total":95995035648,"available":95961899008,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.39200000000000002,"15min":0.40899999999999997},"min":{"1min":0.20499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.48999999999999999,"5min":2.0739999999999998,"15min":2.0739999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.25900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.61499999999999999,"5min":2.008,"15min":2.008},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45500000000000002,"15min":0.47299999999999998},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.55400000000000005,"5min":2.1429999999999998,"15min":2.1429999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.38600000000000001,"15min":0.376},"min":{"1min":0.33300000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.58499999999999996,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.44500000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.54300000000000004,"15min":0.53300000000000003},"min":{"1min":0.432,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.53600000000000003,"15min":0.53000000000000003},"min":{"1min":0.35599999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:04:59 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.53800000000000003,"15min":0.52400000000000002},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.51000000000000001,"15min":0.49199999999999999},"min":{"1min":0.40100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58299999999999996,"15min":0.58099999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.47599999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Aug 17 17:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:05:27.116 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:05:27.116 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:05:27.116 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:05:27.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:05:27.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:26 smithi175 systemd[1]: Stopping Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:05:27.332 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:27 smithi064 ceph-mon[30574]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:27 smithi064 ceph-mon[40964]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:27.566 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-mon[34606]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:27.566 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.266Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:05:27.566 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[61438]: ts=2024-08-17T17:05:27.267Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:05:27.567 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 podman[64661]: 2024-08-17 17:05:27.277142447 +0000 UTC m=+0.138649860 container died 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:27.899 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 podman[64661]: 2024-08-17 17:05:27.665140209 +0000 UTC m=+0.526647618 container cleanup 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:27.900 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 bash[64661]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:05:28.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:27 smithi175 podman[64676]: 2024-08-17 17:05:27.899431494 +0000 UTC m=+0.620539340 container remove 23ba8359dbb2873bd194e244bc08b1028177b8ae8b6bd4617096df4ef74e3b51 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:28.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:05:28.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 systemd[1]: Stopped Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:05:28.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.118s CPU time. 2024-08-17T17:05:28.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 systemd[1]: Starting Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:05:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-mon[30574]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-mon[40964]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:28.497 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-mon[34606]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:05:28.497 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 podman[64769]: 2024-08-17 17:05:28.168961471 +0000 UTC m=+0.019000895 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:05:28.497 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 podman[64769]: 2024-08-17 17:05:28.283886752 +0000 UTC m=+0.133926176 container create a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:28.498 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 podman[64769]: 2024-08-17 17:05:28.494419445 +0000 UTC m=+0.344458880 container init a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:28.498 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 podman[64769]: 2024-08-17 17:05:28.497206268 +0000 UTC m=+0.347245695 container start a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:05:28.761 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Bus STOPPING 2024-08-17T17:05:28.821 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.520Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:05:28.821 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.521Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:05:28.821 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.521Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi175 (none))" 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.521Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.521Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.522Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.522Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.523Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.523Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.525Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.525Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.434µs 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.525Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.527Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.527Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-17T17:05:28.822 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.527Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.58µs wal_replay_duration=1.675125ms wbl_replay_duration=118ns total_replay_duration=1.73002ms 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.528Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.528Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.528Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.548Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.606933ms db_storage=828ns remote_storage=1.382µs web_handler=402ns query_engine=652ns scrape=370.66µs scrape_sd=79.39µs notify=10.263µs notify_sd=11.948µs rules=18.762304ms tracing=5.982µs 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.548Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:05:28.548Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 bash[64769]: a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 2024-08-17T17:05:28.823 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:05:28 smithi175 systemd[1]: Started Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:05:29.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:05:29.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Bus STOPPED 2024-08-17T17:05:29.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Bus STARTING 2024-08-17T17:05:29.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Serving on http://:::9283 2024-08-17T17:05:29.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Bus STARTED 2024-08-17T17:05:29.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:28] ENGINE Bus STOPPING 2024-08-17T17:05:29.209 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:29.209 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:05:29.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:05:29.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STOPPED 2024-08-17T17:05:29.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STARTING 2024-08-17T17:05:29.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Serving on http://:::9283 2024-08-17T17:05:29.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STARTED 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3153521373' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:29.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3153521373' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:05:29.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:29.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:29.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:29 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3153521373' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:05:29.756 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STOPPING 2024-08-17T17:05:30.090 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:05:30.090 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STOPPED 2024-08-17T17:05:30.090 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:29] ENGINE Bus STARTING 2024-08-17T17:05:30.091 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:30] ENGINE Serving on http://:::9283 2024-08-17T17:05:30.091 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:05:30] ENGINE Bus STARTED 2024-08-17T17:05:30.138 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:05:30.139 INFO:tasks.cephadm:Setup complete, yielding 2024-08-17T17:05:30.139 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-17T17:05:30.149 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2024-08-17T17:05:30.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:30.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:30 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[30574]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi175.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:05:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:30 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi175.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:05:30.600 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:31.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:31 smithi064 ceph-mon[30574]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:31.310 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:31 smithi064 ceph-mon[40964]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:31.641 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:31 smithi175 ceph-mon[34606]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[30574]: pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[40964]: pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:33.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:05:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:33 smithi175 ceph-mon[34606]: pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:33 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:05:34.063 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:05:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:05:33.692Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004212675s 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:34 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:34 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[30570]: 2024-08-17T17:05:34.073+0000 7ff200371640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:05:34.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-17T17:05:35.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:05:35.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: fsmap foo:0 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: Saving service mds.foo spec with placement count:2 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:35.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[40964]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: fsmap foo:0 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: Saving service mds.foo spec with placement count:2 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:35.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:35.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:35 smithi064 ceph-mon[30574]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:35.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: fsmap foo:0 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: Saving service mds.foo spec with placement count:2 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:35 smithi175 ceph-mon[34606]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:35.600 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:36 smithi175 ceph-mon[34606]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:05:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:36 smithi064 ceph-mon[30574]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:05:36.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:36 smithi064 ceph-mon[40964]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:05:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:37 smithi175 ceph-mon[34606]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:37.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:37 smithi064 ceph-mon[30574]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:37.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:37 smithi064 ceph-mon[40964]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[30574]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:38 smithi064 ceph-mon[40964]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:38.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-17T17:05:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:38 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:38 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:38 smithi175 ceph-mon[34606]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:39.394 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:40.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: Deploying daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:40.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: Deploying daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: Deploying daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:05:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:40.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:42.117 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:05:41] "GET /metrics HTTP/1.1" 200 33380 "" "Prometheus/2.43.0" 2024-08-17T17:05:42.728 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:05:42.737 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:05:42.738 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:05:42.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:05:43.321 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:43 smithi064 ceph-mon[30574]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:43 smithi064 ceph-mon[40964]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:43 smithi175 ceph-mon[34606]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:boot 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: daemon mds.foo.smithi064.qahkwa assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: Cluster is now healthy 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: fsmap foo:0 1 up:standby 2024-08-17T17:05:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:creating} 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: daemon mds.foo.smithi064.qahkwa is now active in filesystem foo as rank 0 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:44 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:44.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:boot 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: daemon mds.foo.smithi064.qahkwa assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: Cluster is now healthy 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: fsmap foo:0 1 up:standby 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:creating} 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: daemon mds.foo.smithi064.qahkwa is now active in filesystem foo as rank 0 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:boot 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: daemon mds.foo.smithi064.qahkwa assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: Cluster is now healthy 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: fsmap foo:0 1 up:standby 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:creating} 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: daemon mds.foo.smithi064.qahkwa is now active in filesystem foo as rank 0 2024-08-17T17:05:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:44.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:05:44.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:05:44.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:44 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:45.065 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:45.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:45 smithi175 ceph-mon[34606]: Deploying daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:05:45.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:45 smithi175 ceph-mon[34606]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:45.264 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:45 smithi175 ceph-mon[34606]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:45.265 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:45 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 2024-08-17T17:05:45.265 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:45 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2953248474' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:05:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[30574]: Deploying daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:05:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[30574]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[30574]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[30574]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 2024-08-17T17:05:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2953248474' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:05:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[40964]: Deploying daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:05:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[40964]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[40964]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 2024-08-17T17:05:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:45 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2953248474' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:05:45.757 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":21,"flags":0,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":3625104862},{"type":"v1","addr":"172.21.15.64:6801","nonce":3625104862}]},"active_addr":"172.21.15.64:6801/3625104862","active_change":"2024-08-17T17:04:10.166033+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":4139680086}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1135961297}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3398196717}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1031286826}]}]} 2024-08-17T17:05:45.761 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:05:45.762 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:05:45.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:05:45.989 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:boot 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:46 smithi175 ceph-mon[34606]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-08-17T17:05:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:boot 2024-08-17T17:05:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[40964]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:boot 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:47.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:47.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:46 smithi064 ceph-mon[30574]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-08-17T17:05:47.903 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:47.904 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":71,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:05:35.077783+0000","last_up_change":"2024-08-17T17:03:43.486679+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6803","nonce":1608794379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6805","nonce":1608794379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6809","nonce":1608794379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6807","nonce":1608794379}]},"public_addr":"172.21.15.64:6803/1608794379","cluster_addr":"172.21.15.64:6805/1608794379","heartbeat_back_addr":"172.21.15.64:6809/1608794379","heartbeat_front_addr":"172.21.15.64:6807/1608794379","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6811","nonce":2757074067}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6813","nonce":2757074067}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6817","nonce":2757074067}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6815","nonce":2757074067}]},"public_addr":"172.21.15.64:6811/2757074067","cluster_addr":"172.21.15.64:6813/2757074067","heartbeat_back_addr":"172.21.15.64:6817/2757074067","heartbeat_front_addr":"172.21.15.64:6815/2757074067","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6819","nonce":1249488719}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6821","nonce":1249488719}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6825","nonce":1249488719}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6823","nonce":1249488719}]},"public_addr":"172.21.15.64:6819/1249488719","cluster_addr":"172.21.15.64:6821/1249488719","heartbeat_back_addr":"172.21.15.64:6825/1249488719","heartbeat_front_addr":"172.21.15.64:6823/1249488719","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6827","nonce":495130726}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6829","nonce":495130726}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6833","nonce":495130726}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6831","nonce":495130726}]},"public_addr":"172.21.15.64:6827/495130726","cluster_addr":"172.21.15.64:6829/495130726","heartbeat_back_addr":"172.21.15.64:6833/495130726","heartbeat_front_addr":"172.21.15.64:6831/495130726","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:05:49.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[30574]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:49.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[30574]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:49.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/895905280' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:49.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[30574]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:49.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[40964]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:49.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:49.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/895905280' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:49.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:48 smithi064 ceph-mon[40964]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:48 smithi175 ceph-mon[34606]: mds.? [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] up:active 2024-08-17T17:05:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:48 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:48 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/895905280' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:48 smithi175 ceph-mon[34606]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:49.176 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:05:49.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:05:49.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:50.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[30574]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:50.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[40964]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:50.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:50 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:50 smithi175 ceph-mon[34606]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:50.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:50 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:51.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:51 smithi064 ceph-mon[30574]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:standby 2024-08-17T17:05:51.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:51 smithi064 ceph-mon[30574]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:51.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:51 smithi064 ceph-mon[40964]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:standby 2024-08-17T17:05:51.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:51 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:51.566 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:05:51.566 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":71,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:05:35.077783+0000","last_up_change":"2024-08-17T17:03:43.486679+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6803","nonce":1608794379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6805","nonce":1608794379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6809","nonce":1608794379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1608794379},{"type":"v1","addr":"172.21.15.64:6807","nonce":1608794379}]},"public_addr":"172.21.15.64:6803/1608794379","cluster_addr":"172.21.15.64:6805/1608794379","heartbeat_back_addr":"172.21.15.64:6809/1608794379","heartbeat_front_addr":"172.21.15.64:6807/1608794379","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6811","nonce":2757074067}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6813","nonce":2757074067}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6817","nonce":2757074067}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":2757074067},{"type":"v1","addr":"172.21.15.64:6815","nonce":2757074067}]},"public_addr":"172.21.15.64:6811/2757074067","cluster_addr":"172.21.15.64:6813/2757074067","heartbeat_back_addr":"172.21.15.64:6817/2757074067","heartbeat_front_addr":"172.21.15.64:6815/2757074067","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6819","nonce":1249488719}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6821","nonce":1249488719}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6825","nonce":1249488719}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":1249488719},{"type":"v1","addr":"172.21.15.64:6823","nonce":1249488719}]},"public_addr":"172.21.15.64:6819/1249488719","cluster_addr":"172.21.15.64:6821/1249488719","heartbeat_back_addr":"172.21.15.64:6825/1249488719","heartbeat_front_addr":"172.21.15.64:6823/1249488719","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6827","nonce":495130726}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6829","nonce":495130726}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6833","nonce":495130726}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":495130726},{"type":"v1","addr":"172.21.15.64:6831","nonce":495130726}]},"public_addr":"172.21.15.64:6827/495130726","cluster_addr":"172.21.15.64:6829/495130726","heartbeat_back_addr":"172.21.15.64:6833/495130726","heartbeat_front_addr":"172.21.15.64:6831/495130726","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:05:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:51 smithi175 ceph-mon[34606]: mds.? [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] up:standby 2024-08-17T17:05:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:51 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:05:51.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:05:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:05:51] "GET /metrics HTTP/1.1" 200 33380 "" "Prometheus/2.43.0" 2024-08-17T17:05:52.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:05:52.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:05:52.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:05:52.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:05:52.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:05:52.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:05:52.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:05:52.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:05:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:52 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3972922876' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:52 smithi175 ceph-mon[34606]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:52 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3972922876' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:52 smithi064 ceph-mon[30574]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:52 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3972922876' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:05:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:52 smithi064 ceph-mon[40964]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:53.671 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.671 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.673 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:53.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:54.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[30574]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:54 smithi064 ceph-mon[40964]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:54 smithi175 ceph-mon[34606]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:56.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:56 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:56.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:56 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:56.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:56 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:56.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:56 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:56.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:56 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:05:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:56 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:05:57.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:57 smithi064 ceph-mon[30574]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:57.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:57 smithi064 ceph-mon[40964]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:57 smithi175 ceph-mon[34606]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T17:05:59.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:05:59 smithi064 ceph-mon[30574]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:05:59.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:05:59 smithi064 ceph-mon[40964]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:05:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:05:59 smithi175 ceph-mon[34606]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:01.216 INFO:teuthology.orchestra.run.smithi064.stdout:111669149745 2024-08-17T17:06:01.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:06:01.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:01 smithi064 ceph-mon[30574]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:01.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:01 smithi064 ceph-mon[40964]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:01.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:01 smithi175 ceph-mon[34606]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:02.140 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:01] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2024-08-17T17:06:03.292 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:03 smithi064 ceph-mon[30574]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:03.292 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:03 smithi064 ceph-mon[40964]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:03 smithi175 ceph-mon[34606]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:06:04.931 INFO:teuthology.orchestra.run.smithi064.stdout:34359738432 2024-08-17T17:06:04.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:06:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:05 smithi175 ceph-mon[34606]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:05.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:05 smithi064 ceph-mon[30574]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:05.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:05 smithi064 ceph-mon[40964]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:06.085 INFO:teuthology.orchestra.run.smithi064.stdout:167503724584 2024-08-17T17:06:06.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:06:07.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:07 smithi064 ceph-mon[30574]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:07.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:07 smithi064 ceph-mon[40964]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:07.650 INFO:teuthology.orchestra.run.smithi064.stdout:77309411382 2024-08-17T17:06:07.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:06:07.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:07 smithi175 ceph-mon[34606]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:09 smithi064 ceph-mon[30574]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:09.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:09 smithi064 ceph-mon[40964]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:09 smithi175 ceph-mon[34606]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:09.844 INFO:teuthology.orchestra.run.smithi064.stdout:133143986221 2024-08-17T17:06:09.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:06:09.848 INFO:teuthology.orchestra.run.smithi064.stdout:236223201311 2024-08-17T17:06:09.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:06:10.456 INFO:teuthology.orchestra.run.smithi064.stdout:55834574909 2024-08-17T17:06:10.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:06:10.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-08-17T17:06:10.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:10.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-08-17T17:06:10.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-08-17T17:06:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-08-17T17:06:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:10.936 INFO:teuthology.orchestra.run.smithi064.stdout:201863462948 2024-08-17T17:06:10.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:06:10.972 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:10.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:10.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:10.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:10.997 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:11.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[30574]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[30574]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[40964]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:06:11.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:06:11.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-08-17T17:06:11.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-mon[40964]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:06:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:11 smithi175 ceph-mon[34606]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:11 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:06:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:11 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:06:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:11 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-08-17T17:06:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:11 smithi175 ceph-mon[34606]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:06:11.865 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:11] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.43.0" 2024-08-17T17:06:12.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:12 smithi064 ceph-mon[30574]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:06:12.526 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:12 smithi064 ceph-mon[40964]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:06:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:12 smithi175 ceph-mon[34606]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:06:13.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:13 smithi064 ceph-mon[30574]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:13.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:13 smithi064 ceph-mon[40964]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:13 smithi175 ceph-mon[34606]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:13.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:15.254 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:15.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:15 smithi064 ceph-mon[30574]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:15.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:15 smithi064 ceph-mon[40964]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:15 smithi175 ceph-mon[34606]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:17.328 INFO:teuthology.orchestra.run.smithi064.stdout:111669149749 2024-08-17T17:06:17.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:17 smithi064 ceph-mon[30574]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:17.329 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:17 smithi064 ceph-mon[40964]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:17.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:17 smithi175 ceph-mon[34606]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:18 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1934626872' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:06:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:18 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1934626872' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:06:18.656 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:18 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1934626872' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:06:18.803 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149745 got 111669149749 for osd.3 2024-08-17T17:06:18.803 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:19.185 INFO:teuthology.orchestra.run.smithi064.stdout:236223201313 2024-08-17T17:06:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:19 smithi064 ceph-mon[30574]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:19 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1270191497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:06:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:19 smithi064 ceph-mon[40964]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:19 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1270191497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:06:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:19 smithi175 ceph-mon[34606]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:19 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1270191497' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:06:21.343 INFO:teuthology.orchestra.run.smithi064.stdout:55834574911 2024-08-17T17:06:21.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:21 smithi064 ceph-mon[30574]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:21.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:21 smithi064 ceph-mon[40964]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:21 smithi175 ceph-mon[34606]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:21.661 INFO:teuthology.orchestra.run.smithi064.stdout:34359738436 2024-08-17T17:06:21.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:21] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.43.0" 2024-08-17T17:06:22.004 INFO:teuthology.orchestra.run.smithi064.stdout:167503724587 2024-08-17T17:06:22.213 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:22.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1873605565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:06:22.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3852635138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:06:22.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2226956308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:06:22.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[30574]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:22.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1873605565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:06:22.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3852635138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:06:22.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2226956308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:06:22.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:22 smithi064 ceph-mon[40964]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:22.659 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724584 got 167503724587 for osd.5 2024-08-17T17:06:22.659 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:22 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1873605565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:06:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:22 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3852635138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:06:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:22 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2226956308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:06:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:22 smithi175 ceph-mon[34606]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:23.879 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574911 for osd.1 2024-08-17T17:06:23.879 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:23.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:23 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2160378831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:06:23.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:23 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2160378831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:06:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:23 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2160378831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:06:24.424 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738436 for osd.0 2024-08-17T17:06:24.424 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:24.639 INFO:teuthology.orchestra.run.smithi064.stdout:77309411386 2024-08-17T17:06:24.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:24 smithi064 ceph-mon[30574]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:24.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:24 smithi064 ceph-mon[40964]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:24 smithi175 ceph-mon[34606]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:25.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411386 for osd.2 2024-08-17T17:06:25.229 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:25.838 INFO:teuthology.orchestra.run.smithi064.stdout:133143986224 2024-08-17T17:06:25.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:25.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:26.231 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201313 for osd.7 2024-08-17T17:06:26.231 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:26.705 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986221 got 133143986224 for osd.4 2024-08-17T17:06:26.706 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:26 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/715257380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:06:26.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:26 smithi064 ceph-mon[30574]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:26 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/715257380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:06:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:26 smithi064 ceph-mon[40964]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:26 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/715257380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:06:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:26 smithi175 ceph-mon[34606]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:27.798 INFO:teuthology.orchestra.run.smithi064.stdout:201863462953 2024-08-17T17:06:27.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:27 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3282443029' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:06:27.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:27 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3282443029' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:06:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:27 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3282443029' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:06:28.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462953 for osd.6 2024-08-17T17:06:28.661 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:28.661 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:06:28.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:06:28.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:28 smithi064 ceph-mon[30574]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:28.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:28 smithi064 ceph-mon[40964]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:28 smithi175 ceph-mon[34606]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:06:30.658 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:06:31.241 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-08-17T17:06:30.205168+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227972,"kb_used_data":7700,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733244,"statfs":{"total":767960285184,"available":767726841856,"internally_reserved":0,"allocated":7884800,"data_stored":4411926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002446"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260809+0000","last_change":"2024-08-17T17:05:34.260475+0000","last_active":"2024-08-17T17:06:10.260809+0000","last_peered":"2024-08-17T17:06:10.260809+0000","last_clean":"2024-08-17T17:06:10.260809+0000","last_became_active":"2024-08-17T17:05:34.260225+0000","last_became_peered":"2024-08-17T17:05:34.260225+0000","last_unstale":"2024-08-17T17:06:10.260809+0000","last_undegraded":"2024-08-17T17:06:10.260809+0000","last_fullsized":"2024-08-17T17:06:10.260809+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:59.022028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952180+0000","last_change":"2024-08-17T17:05:34.076280+0000","last_active":"2024-08-17T17:06:10.952180+0000","last_peered":"2024-08-17T17:06:10.952180+0000","last_clean":"2024-08-17T17:06:10.952180+0000","last_became_active":"2024-08-17T17:05:34.075941+0000","last_became_peered":"2024-08-17T17:05:34.075941+0000","last_unstale":"2024-08-17T17:06:10.952180+0000","last_undegraded":"2024-08-17T17:06:10.952180+0000","last_fullsized":"2024-08-17T17:06:10.952180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:16.973756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260468+0000","last_change":"2024-08-17T17:05:34.262227+0000","last_active":"2024-08-17T17:06:10.260468+0000","last_peered":"2024-08-17T17:06:10.260468+0000","last_clean":"2024-08-17T17:06:10.260468+0000","last_became_active":"2024-08-17T17:05:34.261893+0000","last_became_peered":"2024-08-17T17:05:34.261893+0000","last_unstale":"2024-08-17T17:06:10.260468+0000","last_undegraded":"2024-08-17T17:06:10.260468+0000","last_fullsized":"2024-08-17T17:06:10.260468+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:43.799464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952562+0000","last_change":"2024-08-17T17:05:34.063483+0000","last_active":"2024-08-17T17:06:10.952562+0000","last_peered":"2024-08-17T17:06:10.952562+0000","last_clean":"2024-08-17T17:06:10.952562+0000","last_became_active":"2024-08-17T17:05:34.062929+0000","last_became_peered":"2024-08-17T17:05:34.062929+0000","last_unstale":"2024-08-17T17:06:10.952562+0000","last_undegraded":"2024-08-17T17:06:10.952562+0000","last_fullsized":"2024-08-17T17:06:10.952562+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:59.125974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952950+0000","last_change":"2024-08-17T17:05:34.262340+0000","last_active":"2024-08-17T17:06:10.952950+0000","last_peered":"2024-08-17T17:06:10.952950+0000","last_clean":"2024-08-17T17:06:10.952950+0000","last_became_active":"2024-08-17T17:05:34.262053+0000","last_became_peered":"2024-08-17T17:05:34.262053+0000","last_unstale":"2024-08-17T17:06:10.952950+0000","last_undegraded":"2024-08-17T17:06:10.952950+0000","last_fullsized":"2024-08-17T17:06:10.952950+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:46:49.902953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.951985+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:06:10.951985+0000","last_peered":"2024-08-17T17:06:10.951985+0000","last_clean":"2024-08-17T17:06:10.951985+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:06:10.951985+0000","last_undegraded":"2024-08-17T17:06:10.951985+0000","last_fullsized":"2024-08-17T17:06:10.951985+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261095+0000","last_change":"2024-08-17T17:05:34.260577+0000","last_active":"2024-08-17T17:06:10.261095+0000","last_peered":"2024-08-17T17:06:10.261095+0000","last_clean":"2024-08-17T17:06:10.261095+0000","last_became_active":"2024-08-17T17:05:34.260159+0000","last_became_peered":"2024-08-17T17:05:34.260159+0000","last_unstale":"2024-08-17T17:06:10.261095+0000","last_undegraded":"2024-08-17T17:06:10.261095+0000","last_fullsized":"2024-08-17T17:06:10.261095+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:22.405353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260754+0000","last_change":"2024-08-17T17:05:34.061630+0000","last_active":"2024-08-17T17:06:10.260754+0000","last_peered":"2024-08-17T17:06:10.260754+0000","last_clean":"2024-08-17T17:06:10.260754+0000","last_became_active":"2024-08-17T17:05:34.061351+0000","last_became_peered":"2024-08-17T17:05:34.061351+0000","last_unstale":"2024-08-17T17:06:10.260754+0000","last_undegraded":"2024-08-17T17:06:10.260754+0000","last_fullsized":"2024-08-17T17:06:10.260754+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:46.229058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952472+0000","last_change":"2024-08-17T17:05:34.073330+0000","last_active":"2024-08-17T17:06:10.952472+0000","last_peered":"2024-08-17T17:06:10.952472+0000","last_clean":"2024-08-17T17:06:10.952472+0000","last_became_active":"2024-08-17T17:05:34.073049+0000","last_became_peered":"2024-08-17T17:05:34.073049+0000","last_unstale":"2024-08-17T17:06:10.952472+0000","last_undegraded":"2024-08-17T17:06:10.952472+0000","last_fullsized":"2024-08-17T17:06:10.952472+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:01.105845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259687+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:06:10.259687+0000","last_peered":"2024-08-17T17:06:10.259687+0000","last_clean":"2024-08-17T17:06:10.259687+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:06:10.259687+0000","last_undegraded":"2024-08-17T17:06:10.259687+0000","last_fullsized":"2024-08-17T17:06:10.259687+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952376+0000","last_change":"2024-08-17T17:05:34.076251+0000","last_active":"2024-08-17T17:06:10.952376+0000","last_peered":"2024-08-17T17:06:10.952376+0000","last_clean":"2024-08-17T17:06:10.952376+0000","last_became_active":"2024-08-17T17:05:34.075935+0000","last_became_peered":"2024-08-17T17:05:34.075935+0000","last_unstale":"2024-08-17T17:06:10.952376+0000","last_undegraded":"2024-08-17T17:06:10.952376+0000","last_fullsized":"2024-08-17T17:06:10.952376+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:36.157157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.256950+0000","last_change":"2024-08-17T17:05:34.099056+0000","last_active":"2024-08-17T17:06:10.256950+0000","last_peered":"2024-08-17T17:06:10.256950+0000","last_clean":"2024-08-17T17:06:10.256950+0000","last_became_active":"2024-08-17T17:05:34.098663+0000","last_became_peered":"2024-08-17T17:05:34.098663+0000","last_unstale":"2024-08-17T17:06:10.256950+0000","last_undegraded":"2024-08-17T17:06:10.256950+0000","last_fullsized":"2024-08-17T17:06:10.256950+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:52:41.809714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952318+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:06:10.952318+0000","last_peered":"2024-08-17T17:06:10.952318+0000","last_clean":"2024-08-17T17:06:10.952318+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:06:10.952318+0000","last_undegraded":"2024-08-17T17:06:10.952318+0000","last_fullsized":"2024-08-17T17:06:10.952318+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259959+0000","last_change":"2024-08-17T17:05:34.260548+0000","last_active":"2024-08-17T17:06:10.259959+0000","last_peered":"2024-08-17T17:06:10.259959+0000","last_clean":"2024-08-17T17:06:10.259959+0000","last_became_active":"2024-08-17T17:05:34.260253+0000","last_became_peered":"2024-08-17T17:05:34.260253+0000","last_unstale":"2024-08-17T17:06:10.259959+0000","last_undegraded":"2024-08-17T17:06:10.259959+0000","last_fullsized":"2024-08-17T17:06:10.259959+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:14.029663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260055+0000","last_change":"2024-08-17T17:05:34.061549+0000","last_active":"2024-08-17T17:06:10.260055+0000","last_peered":"2024-08-17T17:06:10.260055+0000","last_clean":"2024-08-17T17:06:10.260055+0000","last_became_active":"2024-08-17T17:05:34.060643+0000","last_became_peered":"2024-08-17T17:05:34.060643+0000","last_unstale":"2024-08-17T17:06:10.260055+0000","last_undegraded":"2024-08-17T17:06:10.260055+0000","last_fullsized":"2024-08-17T17:06:10.260055+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:22.202930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.953016+0000","last_change":"2024-08-17T17:05:34.099309+0000","last_active":"2024-08-17T17:06:10.953016+0000","last_peered":"2024-08-17T17:06:10.953016+0000","last_clean":"2024-08-17T17:06:10.953016+0000","last_became_active":"2024-08-17T17:05:34.099039+0000","last_became_peered":"2024-08-17T17:05:34.099039+0000","last_unstale":"2024-08-17T17:06:10.953016+0000","last_undegraded":"2024-08-17T17:06:10.953016+0000","last_fullsized":"2024-08-17T17:06:10.953016+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:45.757195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260731+0000","last_change":"2024-08-17T17:05:34.072407+0000","last_active":"2024-08-17T17:06:10.260731+0000","last_peered":"2024-08-17T17:06:10.260731+0000","last_clean":"2024-08-17T17:06:10.260731+0000","last_became_active":"2024-08-17T17:05:34.072051+0000","last_became_peered":"2024-08-17T17:05:34.072051+0000","last_unstale":"2024-08-17T17:06:10.260731+0000","last_undegraded":"2024-08-17T17:06:10.260731+0000","last_fullsized":"2024-08-17T17:06:10.260731+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:33.500831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257528+0000","last_change":"2024-08-17T17:05:34.096056+0000","last_active":"2024-08-17T17:06:10.257528+0000","last_peered":"2024-08-17T17:06:10.257528+0000","last_clean":"2024-08-17T17:06:10.257528+0000","last_became_active":"2024-08-17T17:05:34.095739+0000","last_became_peered":"2024-08-17T17:05:34.095739+0000","last_unstale":"2024-08-17T17:06:10.257528+0000","last_undegraded":"2024-08-17T17:06:10.257528+0000","last_fullsized":"2024-08-17T17:06:10.257528+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:35:37.771456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260106+0000","last_change":"2024-08-17T17:05:34.070221+0000","last_active":"2024-08-17T17:06:10.260106+0000","last_peered":"2024-08-17T17:06:10.260106+0000","last_clean":"2024-08-17T17:06:10.260106+0000","last_became_active":"2024-08-17T17:05:34.069907+0000","last_became_peered":"2024-08-17T17:05:34.069907+0000","last_unstale":"2024-08-17T17:06:10.260106+0000","last_undegraded":"2024-08-17T17:06:10.260106+0000","last_fullsized":"2024-08-17T17:06:10.260106+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:33.534152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257292+0000","last_change":"2024-08-17T17:05:34.105509+0000","last_active":"2024-08-17T17:06:10.257292+0000","last_peered":"2024-08-17T17:06:10.257292+0000","last_clean":"2024-08-17T17:06:10.257292+0000","last_became_active":"2024-08-17T17:05:34.105285+0000","last_became_peered":"2024-08-17T17:05:34.105285+0000","last_unstale":"2024-08-17T17:06:10.257292+0000","last_undegraded":"2024-08-17T17:06:10.257292+0000","last_fullsized":"2024-08-17T17:06:10.257292+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:40.324583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258095+0000","last_change":"2024-08-17T17:05:34.069624+0000","last_active":"2024-08-17T17:06:10.258095+0000","last_peered":"2024-08-17T17:06:10.258095+0000","last_clean":"2024-08-17T17:06:10.258095+0000","last_became_active":"2024-08-17T17:05:34.069331+0000","last_became_peered":"2024-08-17T17:05:34.069331+0000","last_unstale":"2024-08-17T17:06:10.258095+0000","last_undegraded":"2024-08-17T17:06:10.258095+0000","last_fullsized":"2024-08-17T17:06:10.258095+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:51.707707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257189+0000","last_change":"2024-08-17T17:05:34.099184+0000","last_active":"2024-08-17T17:06:10.257189+0000","last_peered":"2024-08-17T17:06:10.257189+0000","last_clean":"2024-08-17T17:06:10.257189+0000","last_became_active":"2024-08-17T17:05:34.098873+0000","last_became_peered":"2024-08-17T17:05:34.098873+0000","last_unstale":"2024-08-17T17:06:10.257189+0000","last_undegraded":"2024-08-17T17:06:10.257189+0000","last_fullsized":"2024-08-17T17:06:10.257189+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:50.555250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260433+0000","last_change":"2024-08-17T17:05:34.070813+0000","last_active":"2024-08-17T17:06:10.260433+0000","last_peered":"2024-08-17T17:06:10.260433+0000","last_clean":"2024-08-17T17:06:10.260433+0000","last_became_active":"2024-08-17T17:05:34.070551+0000","last_became_peered":"2024-08-17T17:05:34.070551+0000","last_unstale":"2024-08-17T17:06:10.260433+0000","last_undegraded":"2024-08-17T17:06:10.260433+0000","last_fullsized":"2024-08-17T17:06:10.260433+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:01.310740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.260015+0000","last_change":"2024-08-17T17:06:11.260015+0000","last_active":"2024-08-17T17:06:11.260015+0000","last_peered":"2024-08-17T17:06:11.260015+0000","last_clean":"2024-08-17T17:06:11.260015+0000","last_became_active":"2024-08-17T17:06:11.259617+0000","last_became_peered":"2024-08-17T17:06:11.259617+0000","last_unstale":"2024-08-17T17:06:11.260015+0000","last_undegraded":"2024-08-17T17:06:11.260015+0000","last_fullsized":"2024-08-17T17:06:11.260015+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:34.685434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.951795+0000","last_change":"2024-08-17T17:03:46.726080+0000","last_active":"2024-08-17T17:06:10.951795+0000","last_peered":"2024-08-17T17:06:10.951795+0000","last_clean":"2024-08-17T17:06:10.951795+0000","last_became_active":"2024-08-17T17:03:46.725689+0000","last_became_peered":"2024-08-17T17:03:46.725689+0000","last_unstale":"2024-08-17T17:06:10.951795+0000","last_undegraded":"2024-08-17T17:06:10.951795+0000","last_fullsized":"2024-08-17T17:06:10.951795+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:54.215266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259864+0000","last_change":"2024-08-17T17:04:29.327959+0000","last_active":"2024-08-17T17:06:10.259864+0000","last_peered":"2024-08-17T17:06:10.259864+0000","last_clean":"2024-08-17T17:06:10.259864+0000","last_became_active":"2024-08-17T17:04:24.310628+0000","last_became_peered":"2024-08-17T17:04:24.310628+0000","last_unstale":"2024-08-17T17:06:10.259864+0000","last_undegraded":"2024-08-17T17:06:10.259864+0000","last_fullsized":"2024-08-17T17:06:10.259864+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:34.789294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083540999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260683+0000","last_change":"2024-08-17T17:05:34.065274+0000","last_active":"2024-08-17T17:06:10.260683+0000","last_peered":"2024-08-17T17:06:10.260683+0000","last_clean":"2024-08-17T17:06:10.260683+0000","last_became_active":"2024-08-17T17:05:34.064845+0000","last_became_peered":"2024-08-17T17:05:34.064845+0000","last_unstale":"2024-08-17T17:06:10.260683+0000","last_undegraded":"2024-08-17T17:06:10.260683+0000","last_fullsized":"2024-08-17T17:06:10.260683+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:19.339757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260351+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:06:10.260351+0000","last_peered":"2024-08-17T17:06:10.260351+0000","last_clean":"2024-08-17T17:06:10.260351+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:06:10.260351+0000","last_undegraded":"2024-08-17T17:06:10.260351+0000","last_fullsized":"2024-08-17T17:06:10.260351+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.864543+0000","last_change":"2024-08-17T17:04:29.455410+0000","last_active":"2024-08-17T17:06:11.864543+0000","last_peered":"2024-08-17T17:06:11.864543+0000","last_clean":"2024-08-17T17:06:11.864543+0000","last_became_active":"2024-08-17T17:04:24.349707+0000","last_became_peered":"2024-08-17T17:04:24.349707+0000","last_unstale":"2024-08-17T17:06:11.864543+0000","last_undegraded":"2024-08-17T17:06:11.864543+0000","last_fullsized":"2024-08-17T17:06:11.864543+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:31.934322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024844199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952170+0000","last_change":"2024-08-17T17:05:34.070784+0000","last_active":"2024-08-17T17:06:10.952170+0000","last_peered":"2024-08-17T17:06:10.952170+0000","last_clean":"2024-08-17T17:06:10.952170+0000","last_became_active":"2024-08-17T17:05:34.070514+0000","last_became_peered":"2024-08-17T17:05:34.070514+0000","last_unstale":"2024-08-17T17:06:10.952170+0000","last_undegraded":"2024-08-17T17:06:10.952170+0000","last_fullsized":"2024-08-17T17:06:10.952170+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:37.677435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952545+0000","last_change":"2024-08-17T17:05:34.099095+0000","last_active":"2024-08-17T17:06:10.952545+0000","last_peered":"2024-08-17T17:06:10.952545+0000","last_clean":"2024-08-17T17:06:10.952545+0000","last_became_active":"2024-08-17T17:05:34.098780+0000","last_became_peered":"2024-08-17T17:05:34.098780+0000","last_unstale":"2024-08-17T17:06:10.952545+0000","last_undegraded":"2024-08-17T17:06:10.952545+0000","last_fullsized":"2024-08-17T17:06:10.952545+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:57.269714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257067+0000","last_change":"2024-08-17T17:04:29.316120+0000","last_active":"2024-08-17T17:06:10.257067+0000","last_peered":"2024-08-17T17:06:10.257067+0000","last_clean":"2024-08-17T17:06:10.257067+0000","last_became_active":"2024-08-17T17:04:24.350215+0000","last_became_peered":"2024-08-17T17:04:24.350215+0000","last_unstale":"2024-08-17T17:06:10.257067+0000","last_undegraded":"2024-08-17T17:06:10.257067+0000","last_fullsized":"2024-08-17T17:06:10.257067+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:49.696633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026442399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257949+0000","last_change":"2024-08-17T17:05:34.069340+0000","last_active":"2024-08-17T17:06:10.257949+0000","last_peered":"2024-08-17T17:06:10.257949+0000","last_clean":"2024-08-17T17:06:10.257949+0000","last_became_active":"2024-08-17T17:05:34.069053+0000","last_became_peered":"2024-08-17T17:05:34.069053+0000","last_unstale":"2024-08-17T17:06:10.257949+0000","last_undegraded":"2024-08-17T17:06:10.257949+0000","last_fullsized":"2024-08-17T17:06:10.257949+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:57.052216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259275+0000","last_change":"2024-08-17T17:05:34.091741+0000","last_active":"2024-08-17T17:06:10.259275+0000","last_peered":"2024-08-17T17:06:10.259275+0000","last_clean":"2024-08-17T17:06:10.259275+0000","last_became_active":"2024-08-17T17:05:34.091270+0000","last_became_peered":"2024-08-17T17:05:34.091270+0000","last_unstale":"2024-08-17T17:06:10.259275+0000","last_undegraded":"2024-08-17T17:06:10.259275+0000","last_fullsized":"2024-08-17T17:06:10.259275+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:55.779788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":28,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259888+0000","last_change":"2024-08-17T17:04:29.327930+0000","last_active":"2024-08-17T17:06:10.259888+0000","last_peered":"2024-08-17T17:06:10.259888+0000","last_clean":"2024-08-17T17:06:10.259888+0000","last_became_active":"2024-08-17T17:04:24.310910+0000","last_became_peered":"2024-08-17T17:04:24.310910+0000","last_unstale":"2024-08-17T17:06:10.259888+0000","last_undegraded":"2024-08-17T17:06:10.259888+0000","last_fullsized":"2024-08-17T17:06:10.259888+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:37.573482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083398900000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257455+0000","last_change":"2024-08-17T17:05:34.070702+0000","last_active":"2024-08-17T17:06:10.257455+0000","last_peered":"2024-08-17T17:06:10.257455+0000","last_clean":"2024-08-17T17:06:10.257455+0000","last_became_active":"2024-08-17T17:05:34.070437+0000","last_became_peered":"2024-08-17T17:05:34.070437+0000","last_unstale":"2024-08-17T17:06:10.257455+0000","last_undegraded":"2024-08-17T17:06:10.257455+0000","last_fullsized":"2024-08-17T17:06:10.257455+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:34.731471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.260585+0000","last_change":"2024-08-17T17:06:11.260585+0000","last_active":"2024-08-17T17:06:11.260585+0000","last_peered":"2024-08-17T17:06:11.260585+0000","last_clean":"2024-08-17T17:06:11.260585+0000","last_became_active":"2024-08-17T17:06:11.260248+0000","last_became_peered":"2024-08-17T17:06:11.260248+0000","last_unstale":"2024-08-17T17:06:11.260585+0000","last_undegraded":"2024-08-17T17:06:11.260585+0000","last_fullsized":"2024-08-17T17:06:11.260585+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:30.335688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260221+0000","last_change":"2024-08-17T17:05:34.064600+0000","last_active":"2024-08-17T17:06:10.260221+0000","last_peered":"2024-08-17T17:06:10.260221+0000","last_clean":"2024-08-17T17:06:10.260221+0000","last_became_active":"2024-08-17T17:05:34.062889+0000","last_became_peered":"2024-08-17T17:05:34.062889+0000","last_unstale":"2024-08-17T17:06:10.260221+0000","last_undegraded":"2024-08-17T17:06:10.260221+0000","last_fullsized":"2024-08-17T17:06:10.260221+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:55.417929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259084+0000","last_change":"2024-08-17T17:04:29.328629+0000","last_active":"2024-08-17T17:06:10.259084+0000","last_peered":"2024-08-17T17:06:10.259084+0000","last_clean":"2024-08-17T17:06:10.259084+0000","last_became_active":"2024-08-17T17:04:24.349917+0000","last_became_peered":"2024-08-17T17:04:24.349917+0000","last_unstale":"2024-08-17T17:06:10.259084+0000","last_undegraded":"2024-08-17T17:06:10.259084+0000","last_fullsized":"2024-08-17T17:06:10.259084+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:53.984346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029370500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258218+0000","last_change":"2024-08-17T17:05:34.097349+0000","last_active":"2024-08-17T17:06:10.258218+0000","last_peered":"2024-08-17T17:06:10.258218+0000","last_clean":"2024-08-17T17:06:10.258218+0000","last_became_active":"2024-08-17T17:05:34.096785+0000","last_became_peered":"2024-08-17T17:05:34.096785+0000","last_unstale":"2024-08-17T17:06:10.258218+0000","last_undegraded":"2024-08-17T17:06:10.258218+0000","last_fullsized":"2024-08-17T17:06:10.258218+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:35.560324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258489+0000","last_change":"2024-08-17T17:05:34.069234+0000","last_active":"2024-08-17T17:06:10.258489+0000","last_peered":"2024-08-17T17:06:10.258489+0000","last_clean":"2024-08-17T17:06:10.258489+0000","last_became_active":"2024-08-17T17:05:34.068970+0000","last_became_peered":"2024-08-17T17:05:34.068970+0000","last_unstale":"2024-08-17T17:06:10.258489+0000","last_undegraded":"2024-08-17T17:06:10.258489+0000","last_fullsized":"2024-08-17T17:06:10.258489+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:36.618330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952364+0000","last_change":"2024-08-17T17:04:29.455481+0000","last_active":"2024-08-17T17:06:10.952364+0000","last_peered":"2024-08-17T17:06:10.952364+0000","last_clean":"2024-08-17T17:06:10.952364+0000","last_became_active":"2024-08-17T17:04:24.350074+0000","last_became_peered":"2024-08-17T17:04:24.350074+0000","last_unstale":"2024-08-17T17:06:10.952364+0000","last_undegraded":"2024-08-17T17:06:10.952364+0000","last_fullsized":"2024-08-17T17:06:10.952364+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:30.046202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025965899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952534+0000","last_change":"2024-08-17T17:05:34.260557+0000","last_active":"2024-08-17T17:06:10.952534+0000","last_peered":"2024-08-17T17:06:10.952534+0000","last_clean":"2024-08-17T17:06:10.952534+0000","last_became_active":"2024-08-17T17:05:34.260254+0000","last_became_peered":"2024-08-17T17:05:34.260254+0000","last_unstale":"2024-08-17T17:06:10.952534+0000","last_undegraded":"2024-08-17T17:06:10.952534+0000","last_fullsized":"2024-08-17T17:06:10.952534+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:07.261500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261176+0000","last_change":"2024-08-17T17:05:34.070409+0000","last_active":"2024-08-17T17:06:10.261176+0000","last_peered":"2024-08-17T17:06:10.261176+0000","last_clean":"2024-08-17T17:06:10.261176+0000","last_became_active":"2024-08-17T17:05:34.070058+0000","last_became_peered":"2024-08-17T17:05:34.070058+0000","last_unstale":"2024-08-17T17:06:10.261176+0000","last_undegraded":"2024-08-17T17:06:10.261176+0000","last_fullsized":"2024-08-17T17:06:10.261176+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:35.963134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259010+0000","last_change":"2024-08-17T17:04:29.328607+0000","last_active":"2024-08-17T17:06:10.259010+0000","last_peered":"2024-08-17T17:06:10.259010+0000","last_clean":"2024-08-17T17:06:10.259010+0000","last_became_active":"2024-08-17T17:04:24.349906+0000","last_became_peered":"2024-08-17T17:04:24.349906+0000","last_unstale":"2024-08-17T17:06:10.259010+0000","last_undegraded":"2024-08-17T17:06:10.259010+0000","last_fullsized":"2024-08-17T17:06:10.259010+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:29.809856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027426399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260289+0000","last_change":"2024-08-17T17:05:34.261203+0000","last_active":"2024-08-17T17:06:10.260289+0000","last_peered":"2024-08-17T17:06:10.260289+0000","last_clean":"2024-08-17T17:06:10.260289+0000","last_became_active":"2024-08-17T17:05:34.260903+0000","last_became_peered":"2024-08-17T17:05:34.260903+0000","last_unstale":"2024-08-17T17:06:10.260289+0000","last_undegraded":"2024-08-17T17:06:10.260289+0000","last_fullsized":"2024-08-17T17:06:10.260289+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:41:59.669408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.953049+0000","last_change":"2024-08-17T17:05:34.070680+0000","last_active":"2024-08-17T17:06:10.953049+0000","last_peered":"2024-08-17T17:06:10.953049+0000","last_clean":"2024-08-17T17:06:10.953049+0000","last_became_active":"2024-08-17T17:05:34.070393+0000","last_became_peered":"2024-08-17T17:05:34.070393+0000","last_unstale":"2024-08-17T17:06:10.953049+0000","last_undegraded":"2024-08-17T17:06:10.953049+0000","last_fullsized":"2024-08-17T17:06:10.953049+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:41.610448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260523+0000","last_change":"2024-08-17T17:04:29.332851+0000","last_active":"2024-08-17T17:06:10.260523+0000","last_peered":"2024-08-17T17:06:10.260523+0000","last_clean":"2024-08-17T17:06:10.260523+0000","last_became_active":"2024-08-17T17:04:24.312005+0000","last_became_peered":"2024-08-17T17:04:24.312005+0000","last_unstale":"2024-08-17T17:06:10.260523+0000","last_undegraded":"2024-08-17T17:06:10.260523+0000","last_fullsized":"2024-08-17T17:06:10.260523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:53.614800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029301899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.256775+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:06:10.256775+0000","last_peered":"2024-08-17T17:06:10.256775+0000","last_clean":"2024-08-17T17:06:10.256775+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:06:10.256775+0000","last_undegraded":"2024-08-17T17:06:10.256775+0000","last_fullsized":"2024-08-17T17:06:10.256775+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257303+0000","last_change":"2024-08-17T17:05:34.099166+0000","last_active":"2024-08-17T17:06:10.257303+0000","last_peered":"2024-08-17T17:06:10.257303+0000","last_clean":"2024-08-17T17:06:10.257303+0000","last_became_active":"2024-08-17T17:05:34.098703+0000","last_became_peered":"2024-08-17T17:05:34.098703+0000","last_unstale":"2024-08-17T17:06:10.257303+0000","last_undegraded":"2024-08-17T17:06:10.257303+0000","last_fullsized":"2024-08-17T17:06:10.257303+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:05:49.884425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257681+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:06:10.257681+0000","last_peered":"2024-08-17T17:06:10.257681+0000","last_clean":"2024-08-17T17:06:10.257681+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:06:10.257681+0000","last_undegraded":"2024-08-17T17:06:10.257681+0000","last_fullsized":"2024-08-17T17:06:10.257681+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258175+0000","last_change":"2024-08-17T17:05:34.099709+0000","last_active":"2024-08-17T17:06:10.258175+0000","last_peered":"2024-08-17T17:06:10.258175+0000","last_clean":"2024-08-17T17:06:10.258175+0000","last_became_active":"2024-08-17T17:05:34.098666+0000","last_became_peered":"2024-08-17T17:05:34.098666+0000","last_unstale":"2024-08-17T17:06:10.258175+0000","last_undegraded":"2024-08-17T17:06:10.258175+0000","last_fullsized":"2024-08-17T17:06:10.258175+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:59:12.380450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952136+0000","last_change":"2024-08-17T17:05:34.074152+0000","last_active":"2024-08-17T17:06:10.952136+0000","last_peered":"2024-08-17T17:06:10.952136+0000","last_clean":"2024-08-17T17:06:10.952136+0000","last_became_active":"2024-08-17T17:05:34.073729+0000","last_became_peered":"2024-08-17T17:05:34.073729+0000","last_unstale":"2024-08-17T17:06:10.952136+0000","last_undegraded":"2024-08-17T17:06:10.952136+0000","last_fullsized":"2024-08-17T17:06:10.952136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:09.916987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260274+0000","last_change":"2024-08-17T17:05:34.070202+0000","last_active":"2024-08-17T17:06:10.260274+0000","last_peered":"2024-08-17T17:06:10.260274+0000","last_clean":"2024-08-17T17:06:10.260274+0000","last_became_active":"2024-08-17T17:05:34.069297+0000","last_became_peered":"2024-08-17T17:05:34.069297+0000","last_unstale":"2024-08-17T17:06:10.260274+0000","last_undegraded":"2024-08-17T17:06:10.260274+0000","last_fullsized":"2024-08-17T17:06:10.260274+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:32.343433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260196+0000","last_change":"2024-08-17T17:05:34.261470+0000","last_active":"2024-08-17T17:06:10.260196+0000","last_peered":"2024-08-17T17:06:10.260196+0000","last_clean":"2024-08-17T17:06:10.260196+0000","last_became_active":"2024-08-17T17:05:34.260973+0000","last_became_peered":"2024-08-17T17:05:34.260973+0000","last_unstale":"2024-08-17T17:06:10.260196+0000","last_undegraded":"2024-08-17T17:06:10.260196+0000","last_fullsized":"2024-08-17T17:06:10.260196+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:15.201788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952820+0000","last_change":"2024-08-17T17:05:34.074865+0000","last_active":"2024-08-17T17:06:10.952820+0000","last_peered":"2024-08-17T17:06:10.952820+0000","last_clean":"2024-08-17T17:06:10.952820+0000","last_became_active":"2024-08-17T17:05:34.074361+0000","last_became_peered":"2024-08-17T17:05:34.074361+0000","last_unstale":"2024-08-17T17:06:10.952820+0000","last_undegraded":"2024-08-17T17:06:10.952820+0000","last_fullsized":"2024-08-17T17:06:10.952820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:14.357688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260189+0000","last_change":"2024-08-17T17:05:34.096604+0000","last_active":"2024-08-17T17:06:10.260189+0000","last_peered":"2024-08-17T17:06:10.260189+0000","last_clean":"2024-08-17T17:06:10.260189+0000","last_became_active":"2024-08-17T17:05:34.095282+0000","last_became_peered":"2024-08-17T17:05:34.095282+0000","last_unstale":"2024-08-17T17:06:10.260189+0000","last_undegraded":"2024-08-17T17:06:10.260189+0000","last_fullsized":"2024-08-17T17:06:10.260189+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:54.951864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260449+0000","last_change":"2024-08-17T17:05:34.099938+0000","last_active":"2024-08-17T17:06:10.260449+0000","last_peered":"2024-08-17T17:06:10.260449+0000","last_clean":"2024-08-17T17:06:10.260449+0000","last_became_active":"2024-08-17T17:05:34.099559+0000","last_became_peered":"2024-08-17T17:05:34.099559+0000","last_unstale":"2024-08-17T17:06:10.260449+0000","last_undegraded":"2024-08-17T17:06:10.260449+0000","last_fullsized":"2024-08-17T17:06:10.260449+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:20.237416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260422+0000","last_change":"2024-08-17T17:05:34.073485+0000","last_active":"2024-08-17T17:06:10.260422+0000","last_peered":"2024-08-17T17:06:10.260422+0000","last_clean":"2024-08-17T17:06:10.260422+0000","last_became_active":"2024-08-17T17:05:34.073214+0000","last_became_peered":"2024-08-17T17:05:34.073214+0000","last_unstale":"2024-08-17T17:06:10.260422+0000","last_undegraded":"2024-08-17T17:06:10.260422+0000","last_fullsized":"2024-08-17T17:06:10.260422+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:41.426938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952867+0000","last_change":"2024-08-17T17:05:34.261376+0000","last_active":"2024-08-17T17:06:10.952867+0000","last_peered":"2024-08-17T17:06:10.952867+0000","last_clean":"2024-08-17T17:06:10.952867+0000","last_became_active":"2024-08-17T17:05:34.261104+0000","last_became_peered":"2024-08-17T17:05:34.261104+0000","last_unstale":"2024-08-17T17:06:10.952867+0000","last_undegraded":"2024-08-17T17:06:10.952867+0000","last_fullsized":"2024-08-17T17:06:10.952867+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:09.996445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258529+0000","last_change":"2024-08-17T17:05:34.067214+0000","last_active":"2024-08-17T17:06:10.258529+0000","last_peered":"2024-08-17T17:06:10.258529+0000","last_clean":"2024-08-17T17:06:10.258529+0000","last_became_active":"2024-08-17T17:05:34.066832+0000","last_became_peered":"2024-08-17T17:05:34.066832+0000","last_unstale":"2024-08-17T17:06:10.258529+0000","last_undegraded":"2024-08-17T17:06:10.258529+0000","last_fullsized":"2024-08-17T17:06:10.258529+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:03.309536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260967+0000","last_change":"2024-08-17T17:05:34.100262+0000","last_active":"2024-08-17T17:06:10.260967+0000","last_peered":"2024-08-17T17:06:10.260967+0000","last_clean":"2024-08-17T17:06:10.260967+0000","last_became_active":"2024-08-17T17:05:34.099984+0000","last_became_peered":"2024-08-17T17:05:34.099984+0000","last_unstale":"2024-08-17T17:06:10.260967+0000","last_undegraded":"2024-08-17T17:06:10.260967+0000","last_fullsized":"2024-08-17T17:06:10.260967+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:31.894964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257119+0000","last_change":"2024-08-17T17:05:34.070037+0000","last_active":"2024-08-17T17:06:10.257119+0000","last_peered":"2024-08-17T17:06:10.257119+0000","last_clean":"2024-08-17T17:06:10.257119+0000","last_became_active":"2024-08-17T17:05:34.069229+0000","last_became_peered":"2024-08-17T17:05:34.069229+0000","last_unstale":"2024-08-17T17:06:10.257119+0000","last_undegraded":"2024-08-17T17:06:10.257119+0000","last_fullsized":"2024-08-17T17:06:10.257119+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:49:22.627947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257256+0000","last_change":"2024-08-17T17:05:34.096779+0000","last_active":"2024-08-17T17:06:10.257256+0000","last_peered":"2024-08-17T17:06:10.257256+0000","last_clean":"2024-08-17T17:06:10.257256+0000","last_became_active":"2024-08-17T17:05:34.096470+0000","last_became_peered":"2024-08-17T17:05:34.096470+0000","last_unstale":"2024-08-17T17:06:10.257256+0000","last_undegraded":"2024-08-17T17:06:10.257256+0000","last_fullsized":"2024-08-17T17:06:10.257256+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:00.344836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952696+0000","last_change":"2024-08-17T17:05:34.071766+0000","last_active":"2024-08-17T17:06:10.952696+0000","last_peered":"2024-08-17T17:06:10.952696+0000","last_clean":"2024-08-17T17:06:10.952696+0000","last_became_active":"2024-08-17T17:05:34.071377+0000","last_became_peered":"2024-08-17T17:05:34.071377+0000","last_unstale":"2024-08-17T17:06:10.952696+0000","last_undegraded":"2024-08-17T17:06:10.952696+0000","last_fullsized":"2024-08-17T17:06:10.952696+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:49.697261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257221+0000","last_change":"2024-08-17T17:05:34.099471+0000","last_active":"2024-08-17T17:06:10.257221+0000","last_peered":"2024-08-17T17:06:10.257221+0000","last_clean":"2024-08-17T17:06:10.257221+0000","last_became_active":"2024-08-17T17:05:34.099159+0000","last_became_peered":"2024-08-17T17:05:34.099159+0000","last_unstale":"2024-08-17T17:06:10.257221+0000","last_undegraded":"2024-08-17T17:06:10.257221+0000","last_fullsized":"2024-08-17T17:06:10.257221+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:30.799877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260180+0000","last_change":"2024-08-17T17:05:34.067217+0000","last_active":"2024-08-17T17:06:10.260180+0000","last_peered":"2024-08-17T17:06:10.260180+0000","last_clean":"2024-08-17T17:06:10.260180+0000","last_became_active":"2024-08-17T17:05:34.066933+0000","last_became_peered":"2024-08-17T17:05:34.066933+0000","last_unstale":"2024-08-17T17:06:10.260180+0000","last_undegraded":"2024-08-17T17:06:10.260180+0000","last_fullsized":"2024-08-17T17:06:10.260180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:56.648864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257887+0000","last_change":"2024-08-17T17:05:34.097611+0000","last_active":"2024-08-17T17:06:10.257887+0000","last_peered":"2024-08-17T17:06:10.257887+0000","last_clean":"2024-08-17T17:06:10.257887+0000","last_became_active":"2024-08-17T17:05:34.097322+0000","last_became_peered":"2024-08-17T17:05:34.097322+0000","last_unstale":"2024-08-17T17:06:10.257887+0000","last_undegraded":"2024-08-17T17:06:10.257887+0000","last_fullsized":"2024-08-17T17:06:10.257887+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:36:45.311214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260125+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:06:10.260125+0000","last_peered":"2024-08-17T17:06:10.260125+0000","last_clean":"2024-08-17T17:06:10.260125+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:06:10.260125+0000","last_undegraded":"2024-08-17T17:06:10.260125+0000","last_fullsized":"2024-08-17T17:06:10.260125+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260476+0000","last_change":"2024-08-17T17:05:34.101996+0000","last_active":"2024-08-17T17:06:10.260476+0000","last_peered":"2024-08-17T17:06:10.260476+0000","last_clean":"2024-08-17T17:06:10.260476+0000","last_became_active":"2024-08-17T17:05:34.101095+0000","last_became_peered":"2024-08-17T17:05:34.101095+0000","last_unstale":"2024-08-17T17:06:10.260476+0000","last_undegraded":"2024-08-17T17:06:10.260476+0000","last_fullsized":"2024-08-17T17:06:10.260476+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:42.003093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261087+0000","last_change":"2024-08-17T17:05:34.069289+0000","last_active":"2024-08-17T17:06:10.261087+0000","last_peered":"2024-08-17T17:06:10.261087+0000","last_clean":"2024-08-17T17:06:10.261087+0000","last_became_active":"2024-08-17T17:05:34.068969+0000","last_became_peered":"2024-08-17T17:05:34.068969+0000","last_unstale":"2024-08-17T17:06:10.261087+0000","last_undegraded":"2024-08-17T17:06:10.261087+0000","last_fullsized":"2024-08-17T17:06:10.261087+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:07:29.328313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.297438+0000","last_change":"2024-08-17T17:06:11.297438+0000","last_active":"2024-08-17T17:06:11.297438+0000","last_peered":"2024-08-17T17:06:11.297438+0000","last_clean":"2024-08-17T17:06:11.297438+0000","last_became_active":"2024-08-17T17:06:11.297134+0000","last_became_peered":"2024-08-17T17:06:11.297134+0000","last_unstale":"2024-08-17T17:06:11.297438+0000","last_undegraded":"2024-08-17T17:06:11.297438+0000","last_fullsized":"2024-08-17T17:06:11.297438+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:51.294632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952797+0000","last_change":"2024-08-17T17:05:34.066388+0000","last_active":"2024-08-17T17:06:10.952797+0000","last_peered":"2024-08-17T17:06:10.952797+0000","last_clean":"2024-08-17T17:06:10.952797+0000","last_became_active":"2024-08-17T17:05:34.066076+0000","last_became_peered":"2024-08-17T17:05:34.066076+0000","last_unstale":"2024-08-17T17:06:10.952797+0000","last_undegraded":"2024-08-17T17:06:10.952797+0000","last_fullsized":"2024-08-17T17:06:10.952797+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:04.886951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201315,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1372160,"data_stored":921386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.93799999999999994,"15min":1.0029999999999999},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71199999999999997,"5min":13.132999999999999,"15min":13.132999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.41999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71899999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.40100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75700000000000001,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":1.028,"15min":1.028},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66700000000000004,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.135,"15min":1.2290000000000001},"min":{"1min":0.42399999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70099999999999996,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.316}]},{"osd":3,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.44700000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69299999999999995,"5min":0.878,"15min":0.878},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.42499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73199999999999998,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.437}]},{"osd":4,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66100000000000003,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.48699999999999999,"15min":0.49299999999999999},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.61599999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.315,"5min":0.309,"15min":0.309},"max":{"1min":0.61099999999999999,"5min":1.044,"15min":1.044},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66200000000000003,"5min":1.091,"15min":1.091},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.69499999999999995,"5min":1.117,"15min":1.117},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.52300000000000002,"15min":0.52800000000000002},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68899999999999995,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.44900000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":920994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.52600000000000002,"15min":0.51900000000000002},"min":{"1min":0.45500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.68700000000000006,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.94299999999999995,"15min":0.67000000000000004},"min":{"1min":0.45600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.75700000000000001,"5min":19.114999999999998,"15min":19.114999999999998},"last":0.60099999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.437,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.41699999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.82699999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.40799999999999997,"5min":0.315,"15min":0.315},"max":{"1min":0.79900000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.53600000000000003,"15min":0.52600000000000002},"min":{"1min":0.42499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.48099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":0.78300000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.55300000000000005,"15min":0.54500000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63200000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.40899999999999997,"15min":0.39300000000000002},"min":{"1min":0.28199999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.437,"15min":0.41699999999999998},"min":{"1min":0.39500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.439,"15min":0.42099999999999999},"min":{"1min":0.29599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.47799999999999998,"15min":0.45900000000000002},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.374}]},{"osd":7,"last update":"Sat Aug 17 17:05:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.497,"15min":0.49099999999999999},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.49399999999999999,"15min":0.48999999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41399999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717388,"statfs":{"total":95995035648,"available":95966605312,"internally_reserved":0,"allocated":761856,"data_stored":329909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.377,"5min":0.308,"15min":0.308},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.52100000000000002,"15min":0.52200000000000002},"min":{"1min":0.35399999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.68600000000000005,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.48199999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71699999999999997,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.50600000000000001,"15min":0.496},"min":{"1min":0.36699999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74299999999999999,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75700000000000001,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.435},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.40999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81499999999999995,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.434,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73299999999999998,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.503},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.38500000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.79600000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.373},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.59199999999999997,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.41399999999999998,"15min":0.39100000000000001},"min":{"1min":0.312,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65400000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42499999999999999,"15min":0.40300000000000002},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63200000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46400000000000002,"15min":0.44},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.63,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.46600000000000003,"15min":0.46200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.52400000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":330956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.073,"15min":1.323},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69799999999999995,"5min":16.233000000000001,"15min":16.233000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.51500000000000001,"15min":0.52300000000000002},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.65900000000000003,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.056,"15min":1.306},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67600000000000005,"5min":16.245000000000001,"15min":16.245000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.55500000000000005,"15min":0.57299999999999995},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69299999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.69499999999999995,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53600000000000003,"15min":0.54600000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.0600000000000001,"15min":1.3},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.63800000000000001,"5min":16.27,"15min":16.27},"last":0.499}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.40799999999999997,"15min":0.39000000000000001},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.46800000000000003}]},{"osd":6,"last update":"Sat Aug 17 17:06:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.40400000000000003,"15min":0.38700000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.57899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42199999999999999,"15min":0.40200000000000002},"min":{"1min":0.36199999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:05:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.46500000000000002,"15min":0.45500000000000002},"min":{"1min":0.38900000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.44800000000000001,"15min":0.432},"min":{"1min":0.34499999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40899999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":753664,"data_stored":329336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.376,"15min":0.38900000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.495},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40100000000000002,"15min":0.41499999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61799999999999999,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41499999999999998,"15min":0.436},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.67700000000000005,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39100000000000001,"15min":0.41099999999999998},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.437}]},{"osd":2,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.43099999999999999,"15min":0.45100000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54100000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41899999999999998,"15min":0.435},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60799999999999998,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65300000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.82899999999999996,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.80600000000000005}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77500000000000002,"5min":0.877,"15min":0.877},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58999999999999997,"15min":0.59899999999999998},"min":{"1min":0.45800000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.72599999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76800000000000002,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.621,"5min":0.61299999999999999,"15min":0.61099999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.80200000000000005,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.78400000000000003}]},{"osd":7,"last update":"Sat Aug 17 17:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.59499999999999997,"15min":0.59399999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.748},{"interface":"front","average":{"1min":0.626,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.45800000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.83699999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":757760,"data_stored":329862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.34000000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.36399999999999999,"15min":0.34999999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35299999999999998,"15min":0.34100000000000003},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51700000000000002,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.317},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.439}]},{"osd":3,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.44,"15min":0.44},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.504,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.40899999999999997,"15min":0.42299999999999999},"min":{"1min":0.29699999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.496,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.42199999999999999}]},{"osd":4,"last update":"Sat Aug 17 17:05:35 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68999999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58799999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.83699999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59499999999999997,"15min":0.60399999999999998},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.75},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.74299999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:05:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77600000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Aug 17 17:05:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.437,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78900000000000003,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.623,"5min":0.63500000000000001,"15min":0.63700000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.67200000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":328992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:29 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.35399999999999998,"15min":0.32000000000000001},"min":{"1min":0.18099999999999999,"5min":0.155,"15min":0.155},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.36299999999999999,"15min":0.32300000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.76400000000000001,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Aug 17 17:05:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.376},"min":{"1min":0.255,"5min":0.245,"15min":0.245},"max":{"1min":0.57299999999999995,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.38},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.54400000000000004,"5min":1.054,"15min":1.054},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.44700000000000001,"15min":0.437},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57599999999999996,"15min":0.56200000000000006},"min":{"1min":0.36499999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59599999999999997,"15min":0.59199999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.71699999999999997}]},{"osd":5,"last update":"Sat Aug 17 17:06:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.81999999999999995,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78700000000000003,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.626}]},{"osd":6,"last update":"Sat Aug 17 17:05:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.64800000000000002,"15min":0.66400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59099999999999997,"15min":0.59799999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.63800000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42899999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78400000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.41499999999999998,"5min":0.375,"15min":0.375},"max":{"1min":0.85099999999999998,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.68000000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32440,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712712,"statfs":{"total":95995035648,"available":95961817088,"internally_reserved":0,"allocated":1355776,"data_stored":920491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:05:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.375,"15min":0.40300000000000002},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":0.46600000000000003,"5min":2.0739999999999998,"15min":2.0739999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57099999999999995,"5min":2.008,"15min":2.008},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Aug 17 17:05:53 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.42899999999999999,"15min":0.46400000000000002},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.53500000000000003,"5min":2.1429999999999998,"15min":2.1429999999999998},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.48499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.372},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.32000000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:05:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.80600000000000005,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53500000000000003},"min":{"1min":0.38100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.52700000000000002,"15min":0.497},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76100000000000001,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:05:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58299999999999996,"15min":0.58099999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.47599999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.59799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:06:31.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:06:31.459 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:31.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:31 smithi064 ceph-mon[30574]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:31.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:31 smithi064 ceph-mon[40964]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:31.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:31 smithi175 ceph-mon[34606]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:31.829 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:31 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:31] "GET /metrics HTTP/1.1" 200 33925 "" "Prometheus/2.43.0" 2024-08-17T17:06:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:32 smithi064 ceph-mon[30574]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:32.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:32 smithi064 ceph-mon[40964]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:32 smithi175 ceph-mon[34606]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:33.306 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:06:33.307 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:06:33.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:33 smithi064 ceph-mon[30574]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:33.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:33 smithi064 ceph-mon[40964]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:33 smithi175 ceph-mon[34606]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-08-17T17:06:32.205612+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227972,"kb_used_data":7700,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733244,"statfs":{"total":767960285184,"available":767726841856,"internally_reserved":0,"allocated":7884800,"data_stored":4411926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002540"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260809+0000","last_change":"2024-08-17T17:05:34.260475+0000","last_active":"2024-08-17T17:06:10.260809+0000","last_peered":"2024-08-17T17:06:10.260809+0000","last_clean":"2024-08-17T17:06:10.260809+0000","last_became_active":"2024-08-17T17:05:34.260225+0000","last_became_peered":"2024-08-17T17:05:34.260225+0000","last_unstale":"2024-08-17T17:06:10.260809+0000","last_undegraded":"2024-08-17T17:06:10.260809+0000","last_fullsized":"2024-08-17T17:06:10.260809+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:59.022028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952180+0000","last_change":"2024-08-17T17:05:34.076280+0000","last_active":"2024-08-17T17:06:10.952180+0000","last_peered":"2024-08-17T17:06:10.952180+0000","last_clean":"2024-08-17T17:06:10.952180+0000","last_became_active":"2024-08-17T17:05:34.075941+0000","last_became_peered":"2024-08-17T17:05:34.075941+0000","last_unstale":"2024-08-17T17:06:10.952180+0000","last_undegraded":"2024-08-17T17:06:10.952180+0000","last_fullsized":"2024-08-17T17:06:10.952180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:16.973756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260468+0000","last_change":"2024-08-17T17:05:34.262227+0000","last_active":"2024-08-17T17:06:10.260468+0000","last_peered":"2024-08-17T17:06:10.260468+0000","last_clean":"2024-08-17T17:06:10.260468+0000","last_became_active":"2024-08-17T17:05:34.261893+0000","last_became_peered":"2024-08-17T17:05:34.261893+0000","last_unstale":"2024-08-17T17:06:10.260468+0000","last_undegraded":"2024-08-17T17:06:10.260468+0000","last_fullsized":"2024-08-17T17:06:10.260468+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:43.799464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952562+0000","last_change":"2024-08-17T17:05:34.063483+0000","last_active":"2024-08-17T17:06:10.952562+0000","last_peered":"2024-08-17T17:06:10.952562+0000","last_clean":"2024-08-17T17:06:10.952562+0000","last_became_active":"2024-08-17T17:05:34.062929+0000","last_became_peered":"2024-08-17T17:05:34.062929+0000","last_unstale":"2024-08-17T17:06:10.952562+0000","last_undegraded":"2024-08-17T17:06:10.952562+0000","last_fullsized":"2024-08-17T17:06:10.952562+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:59.125974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952950+0000","last_change":"2024-08-17T17:05:34.262340+0000","last_active":"2024-08-17T17:06:10.952950+0000","last_peered":"2024-08-17T17:06:10.952950+0000","last_clean":"2024-08-17T17:06:10.952950+0000","last_became_active":"2024-08-17T17:05:34.262053+0000","last_became_peered":"2024-08-17T17:05:34.262053+0000","last_unstale":"2024-08-17T17:06:10.952950+0000","last_undegraded":"2024-08-17T17:06:10.952950+0000","last_fullsized":"2024-08-17T17:06:10.952950+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:46:49.902953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.951985+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:06:10.951985+0000","last_peered":"2024-08-17T17:06:10.951985+0000","last_clean":"2024-08-17T17:06:10.951985+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:06:10.951985+0000","last_undegraded":"2024-08-17T17:06:10.951985+0000","last_fullsized":"2024-08-17T17:06:10.951985+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261095+0000","last_change":"2024-08-17T17:05:34.260577+0000","last_active":"2024-08-17T17:06:10.261095+0000","last_peered":"2024-08-17T17:06:10.261095+0000","last_clean":"2024-08-17T17:06:10.261095+0000","last_became_active":"2024-08-17T17:05:34.260159+0000","last_became_peered":"2024-08-17T17:05:34.260159+0000","last_unstale":"2024-08-17T17:06:10.261095+0000","last_undegraded":"2024-08-17T17:06:10.261095+0000","last_fullsized":"2024-08-17T17:06:10.261095+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:22.405353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260754+0000","last_change":"2024-08-17T17:05:34.061630+0000","last_active":"2024-08-17T17:06:10.260754+0000","last_peered":"2024-08-17T17:06:10.260754+0000","last_clean":"2024-08-17T17:06:10.260754+0000","last_became_active":"2024-08-17T17:05:34.061351+0000","last_became_peered":"2024-08-17T17:05:34.061351+0000","last_unstale":"2024-08-17T17:06:10.260754+0000","last_undegraded":"2024-08-17T17:06:10.260754+0000","last_fullsized":"2024-08-17T17:06:10.260754+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:46.229058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952472+0000","last_change":"2024-08-17T17:05:34.073330+0000","last_active":"2024-08-17T17:06:10.952472+0000","last_peered":"2024-08-17T17:06:10.952472+0000","last_clean":"2024-08-17T17:06:10.952472+0000","last_became_active":"2024-08-17T17:05:34.073049+0000","last_became_peered":"2024-08-17T17:05:34.073049+0000","last_unstale":"2024-08-17T17:06:10.952472+0000","last_undegraded":"2024-08-17T17:06:10.952472+0000","last_fullsized":"2024-08-17T17:06:10.952472+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:01.105845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259687+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:06:10.259687+0000","last_peered":"2024-08-17T17:06:10.259687+0000","last_clean":"2024-08-17T17:06:10.259687+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:06:10.259687+0000","last_undegraded":"2024-08-17T17:06:10.259687+0000","last_fullsized":"2024-08-17T17:06:10.259687+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952376+0000","last_change":"2024-08-17T17:05:34.076251+0000","last_active":"2024-08-17T17:06:10.952376+0000","last_peered":"2024-08-17T17:06:10.952376+0000","last_clean":"2024-08-17T17:06:10.952376+0000","last_became_active":"2024-08-17T17:05:34.075935+0000","last_became_peered":"2024-08-17T17:05:34.075935+0000","last_unstale":"2024-08-17T17:06:10.952376+0000","last_undegraded":"2024-08-17T17:06:10.952376+0000","last_fullsized":"2024-08-17T17:06:10.952376+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:36.157157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.256950+0000","last_change":"2024-08-17T17:05:34.099056+0000","last_active":"2024-08-17T17:06:10.256950+0000","last_peered":"2024-08-17T17:06:10.256950+0000","last_clean":"2024-08-17T17:06:10.256950+0000","last_became_active":"2024-08-17T17:05:34.098663+0000","last_became_peered":"2024-08-17T17:05:34.098663+0000","last_unstale":"2024-08-17T17:06:10.256950+0000","last_undegraded":"2024-08-17T17:06:10.256950+0000","last_fullsized":"2024-08-17T17:06:10.256950+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:52:41.809714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952318+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:06:10.952318+0000","last_peered":"2024-08-17T17:06:10.952318+0000","last_clean":"2024-08-17T17:06:10.952318+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:06:10.952318+0000","last_undegraded":"2024-08-17T17:06:10.952318+0000","last_fullsized":"2024-08-17T17:06:10.952318+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259959+0000","last_change":"2024-08-17T17:05:34.260548+0000","last_active":"2024-08-17T17:06:10.259959+0000","last_peered":"2024-08-17T17:06:10.259959+0000","last_clean":"2024-08-17T17:06:10.259959+0000","last_became_active":"2024-08-17T17:05:34.260253+0000","last_became_peered":"2024-08-17T17:05:34.260253+0000","last_unstale":"2024-08-17T17:06:10.259959+0000","last_undegraded":"2024-08-17T17:06:10.259959+0000","last_fullsized":"2024-08-17T17:06:10.259959+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:14.029663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260055+0000","last_change":"2024-08-17T17:05:34.061549+0000","last_active":"2024-08-17T17:06:10.260055+0000","last_peered":"2024-08-17T17:06:10.260055+0000","last_clean":"2024-08-17T17:06:10.260055+0000","last_became_active":"2024-08-17T17:05:34.060643+0000","last_became_peered":"2024-08-17T17:05:34.060643+0000","last_unstale":"2024-08-17T17:06:10.260055+0000","last_undegraded":"2024-08-17T17:06:10.260055+0000","last_fullsized":"2024-08-17T17:06:10.260055+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:22.202930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.953016+0000","last_change":"2024-08-17T17:05:34.099309+0000","last_active":"2024-08-17T17:06:10.953016+0000","last_peered":"2024-08-17T17:06:10.953016+0000","last_clean":"2024-08-17T17:06:10.953016+0000","last_became_active":"2024-08-17T17:05:34.099039+0000","last_became_peered":"2024-08-17T17:05:34.099039+0000","last_unstale":"2024-08-17T17:06:10.953016+0000","last_undegraded":"2024-08-17T17:06:10.953016+0000","last_fullsized":"2024-08-17T17:06:10.953016+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:45.757195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260731+0000","last_change":"2024-08-17T17:05:34.072407+0000","last_active":"2024-08-17T17:06:10.260731+0000","last_peered":"2024-08-17T17:06:10.260731+0000","last_clean":"2024-08-17T17:06:10.260731+0000","last_became_active":"2024-08-17T17:05:34.072051+0000","last_became_peered":"2024-08-17T17:05:34.072051+0000","last_unstale":"2024-08-17T17:06:10.260731+0000","last_undegraded":"2024-08-17T17:06:10.260731+0000","last_fullsized":"2024-08-17T17:06:10.260731+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:33.500831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257528+0000","last_change":"2024-08-17T17:05:34.096056+0000","last_active":"2024-08-17T17:06:10.257528+0000","last_peered":"2024-08-17T17:06:10.257528+0000","last_clean":"2024-08-17T17:06:10.257528+0000","last_became_active":"2024-08-17T17:05:34.095739+0000","last_became_peered":"2024-08-17T17:05:34.095739+0000","last_unstale":"2024-08-17T17:06:10.257528+0000","last_undegraded":"2024-08-17T17:06:10.257528+0000","last_fullsized":"2024-08-17T17:06:10.257528+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:35:37.771456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260106+0000","last_change":"2024-08-17T17:05:34.070221+0000","last_active":"2024-08-17T17:06:10.260106+0000","last_peered":"2024-08-17T17:06:10.260106+0000","last_clean":"2024-08-17T17:06:10.260106+0000","last_became_active":"2024-08-17T17:05:34.069907+0000","last_became_peered":"2024-08-17T17:05:34.069907+0000","last_unstale":"2024-08-17T17:06:10.260106+0000","last_undegraded":"2024-08-17T17:06:10.260106+0000","last_fullsized":"2024-08-17T17:06:10.260106+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:33.534152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257292+0000","last_change":"2024-08-17T17:05:34.105509+0000","last_active":"2024-08-17T17:06:10.257292+0000","last_peered":"2024-08-17T17:06:10.257292+0000","last_clean":"2024-08-17T17:06:10.257292+0000","last_became_active":"2024-08-17T17:05:34.105285+0000","last_became_peered":"2024-08-17T17:05:34.105285+0000","last_unstale":"2024-08-17T17:06:10.257292+0000","last_undegraded":"2024-08-17T17:06:10.257292+0000","last_fullsized":"2024-08-17T17:06:10.257292+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:40.324583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258095+0000","last_change":"2024-08-17T17:05:34.069624+0000","last_active":"2024-08-17T17:06:10.258095+0000","last_peered":"2024-08-17T17:06:10.258095+0000","last_clean":"2024-08-17T17:06:10.258095+0000","last_became_active":"2024-08-17T17:05:34.069331+0000","last_became_peered":"2024-08-17T17:05:34.069331+0000","last_unstale":"2024-08-17T17:06:10.258095+0000","last_undegraded":"2024-08-17T17:06:10.258095+0000","last_fullsized":"2024-08-17T17:06:10.258095+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:51.707707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257189+0000","last_change":"2024-08-17T17:05:34.099184+0000","last_active":"2024-08-17T17:06:10.257189+0000","last_peered":"2024-08-17T17:06:10.257189+0000","last_clean":"2024-08-17T17:06:10.257189+0000","last_became_active":"2024-08-17T17:05:34.098873+0000","last_became_peered":"2024-08-17T17:05:34.098873+0000","last_unstale":"2024-08-17T17:06:10.257189+0000","last_undegraded":"2024-08-17T17:06:10.257189+0000","last_fullsized":"2024-08-17T17:06:10.257189+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:50.555250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260433+0000","last_change":"2024-08-17T17:05:34.070813+0000","last_active":"2024-08-17T17:06:10.260433+0000","last_peered":"2024-08-17T17:06:10.260433+0000","last_clean":"2024-08-17T17:06:10.260433+0000","last_became_active":"2024-08-17T17:05:34.070551+0000","last_became_peered":"2024-08-17T17:05:34.070551+0000","last_unstale":"2024-08-17T17:06:10.260433+0000","last_undegraded":"2024-08-17T17:06:10.260433+0000","last_fullsized":"2024-08-17T17:06:10.260433+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:01.310740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.260015+0000","last_change":"2024-08-17T17:06:11.260015+0000","last_active":"2024-08-17T17:06:11.260015+0000","last_peered":"2024-08-17T17:06:11.260015+0000","last_clean":"2024-08-17T17:06:11.260015+0000","last_became_active":"2024-08-17T17:06:11.259617+0000","last_became_peered":"2024-08-17T17:06:11.259617+0000","last_unstale":"2024-08-17T17:06:11.260015+0000","last_undegraded":"2024-08-17T17:06:11.260015+0000","last_fullsized":"2024-08-17T17:06:11.260015+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:34.685434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.951795+0000","last_change":"2024-08-17T17:03:46.726080+0000","last_active":"2024-08-17T17:06:10.951795+0000","last_peered":"2024-08-17T17:06:10.951795+0000","last_clean":"2024-08-17T17:06:10.951795+0000","last_became_active":"2024-08-17T17:03:46.725689+0000","last_became_peered":"2024-08-17T17:03:46.725689+0000","last_unstale":"2024-08-17T17:06:10.951795+0000","last_undegraded":"2024-08-17T17:06:10.951795+0000","last_fullsized":"2024-08-17T17:06:10.951795+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:54.215266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.259520+0000","last_change":"2024-08-17T17:04:29.327959+0000","last_active":"2024-08-17T17:06:11.259520+0000","last_peered":"2024-08-17T17:06:11.259520+0000","last_clean":"2024-08-17T17:06:11.259520+0000","last_became_active":"2024-08-17T17:04:24.310628+0000","last_became_peered":"2024-08-17T17:04:24.310628+0000","last_unstale":"2024-08-17T17:06:11.259520+0000","last_undegraded":"2024-08-17T17:06:11.259520+0000","last_fullsized":"2024-08-17T17:06:11.259520+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:34.789294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083540999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260683+0000","last_change":"2024-08-17T17:05:34.065274+0000","last_active":"2024-08-17T17:06:10.260683+0000","last_peered":"2024-08-17T17:06:10.260683+0000","last_clean":"2024-08-17T17:06:10.260683+0000","last_became_active":"2024-08-17T17:05:34.064845+0000","last_became_peered":"2024-08-17T17:05:34.064845+0000","last_unstale":"2024-08-17T17:06:10.260683+0000","last_undegraded":"2024-08-17T17:06:10.260683+0000","last_fullsized":"2024-08-17T17:06:10.260683+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:19.339757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260351+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:06:10.260351+0000","last_peered":"2024-08-17T17:06:10.260351+0000","last_clean":"2024-08-17T17:06:10.260351+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:06:10.260351+0000","last_undegraded":"2024-08-17T17:06:10.260351+0000","last_fullsized":"2024-08-17T17:06:10.260351+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.864543+0000","last_change":"2024-08-17T17:04:29.455410+0000","last_active":"2024-08-17T17:06:11.864543+0000","last_peered":"2024-08-17T17:06:11.864543+0000","last_clean":"2024-08-17T17:06:11.864543+0000","last_became_active":"2024-08-17T17:04:24.349707+0000","last_became_peered":"2024-08-17T17:04:24.349707+0000","last_unstale":"2024-08-17T17:06:11.864543+0000","last_undegraded":"2024-08-17T17:06:11.864543+0000","last_fullsized":"2024-08-17T17:06:11.864543+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:31.934322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024844199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952170+0000","last_change":"2024-08-17T17:05:34.070784+0000","last_active":"2024-08-17T17:06:10.952170+0000","last_peered":"2024-08-17T17:06:10.952170+0000","last_clean":"2024-08-17T17:06:10.952170+0000","last_became_active":"2024-08-17T17:05:34.070514+0000","last_became_peered":"2024-08-17T17:05:34.070514+0000","last_unstale":"2024-08-17T17:06:10.952170+0000","last_undegraded":"2024-08-17T17:06:10.952170+0000","last_fullsized":"2024-08-17T17:06:10.952170+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:37.677435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952545+0000","last_change":"2024-08-17T17:05:34.099095+0000","last_active":"2024-08-17T17:06:10.952545+0000","last_peered":"2024-08-17T17:06:10.952545+0000","last_clean":"2024-08-17T17:06:10.952545+0000","last_became_active":"2024-08-17T17:05:34.098780+0000","last_became_peered":"2024-08-17T17:05:34.098780+0000","last_unstale":"2024-08-17T17:06:10.952545+0000","last_undegraded":"2024-08-17T17:06:10.952545+0000","last_fullsized":"2024-08-17T17:06:10.952545+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:57.269714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.257774+0000","last_change":"2024-08-17T17:04:29.316120+0000","last_active":"2024-08-17T17:06:11.257774+0000","last_peered":"2024-08-17T17:06:11.257774+0000","last_clean":"2024-08-17T17:06:11.257774+0000","last_became_active":"2024-08-17T17:04:24.350215+0000","last_became_peered":"2024-08-17T17:04:24.350215+0000","last_unstale":"2024-08-17T17:06:11.257774+0000","last_undegraded":"2024-08-17T17:06:11.257774+0000","last_fullsized":"2024-08-17T17:06:11.257774+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:49.696633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026442399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257949+0000","last_change":"2024-08-17T17:05:34.069340+0000","last_active":"2024-08-17T17:06:10.257949+0000","last_peered":"2024-08-17T17:06:10.257949+0000","last_clean":"2024-08-17T17:06:10.257949+0000","last_became_active":"2024-08-17T17:05:34.069053+0000","last_became_peered":"2024-08-17T17:05:34.069053+0000","last_unstale":"2024-08-17T17:06:10.257949+0000","last_undegraded":"2024-08-17T17:06:10.257949+0000","last_fullsized":"2024-08-17T17:06:10.257949+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:57.052216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259275+0000","last_change":"2024-08-17T17:05:34.091741+0000","last_active":"2024-08-17T17:06:10.259275+0000","last_peered":"2024-08-17T17:06:10.259275+0000","last_clean":"2024-08-17T17:06:10.259275+0000","last_became_active":"2024-08-17T17:05:34.091270+0000","last_became_peered":"2024-08-17T17:05:34.091270+0000","last_unstale":"2024-08-17T17:06:10.259275+0000","last_undegraded":"2024-08-17T17:06:10.259275+0000","last_fullsized":"2024-08-17T17:06:10.259275+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:55.779788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.259401+0000","last_change":"2024-08-17T17:04:29.327930+0000","last_active":"2024-08-17T17:06:11.259401+0000","last_peered":"2024-08-17T17:06:11.259401+0000","last_clean":"2024-08-17T17:06:11.259401+0000","last_became_active":"2024-08-17T17:04:24.310910+0000","last_became_peered":"2024-08-17T17:04:24.310910+0000","last_unstale":"2024-08-17T17:06:11.259401+0000","last_undegraded":"2024-08-17T17:06:11.259401+0000","last_fullsized":"2024-08-17T17:06:11.259401+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:37.573482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083398900000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257455+0000","last_change":"2024-08-17T17:05:34.070702+0000","last_active":"2024-08-17T17:06:10.257455+0000","last_peered":"2024-08-17T17:06:10.257455+0000","last_clean":"2024-08-17T17:06:10.257455+0000","last_became_active":"2024-08-17T17:05:34.070437+0000","last_became_peered":"2024-08-17T17:05:34.070437+0000","last_unstale":"2024-08-17T17:06:10.257455+0000","last_undegraded":"2024-08-17T17:06:10.257455+0000","last_fullsized":"2024-08-17T17:06:10.257455+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:34.731471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.260585+0000","last_change":"2024-08-17T17:06:11.260585+0000","last_active":"2024-08-17T17:06:11.260585+0000","last_peered":"2024-08-17T17:06:11.260585+0000","last_clean":"2024-08-17T17:06:11.260585+0000","last_became_active":"2024-08-17T17:06:11.260248+0000","last_became_peered":"2024-08-17T17:06:11.260248+0000","last_unstale":"2024-08-17T17:06:11.260585+0000","last_undegraded":"2024-08-17T17:06:11.260585+0000","last_fullsized":"2024-08-17T17:06:11.260585+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:30.335688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260221+0000","last_change":"2024-08-17T17:05:34.064600+0000","last_active":"2024-08-17T17:06:10.260221+0000","last_peered":"2024-08-17T17:06:10.260221+0000","last_clean":"2024-08-17T17:06:10.260221+0000","last_became_active":"2024-08-17T17:05:34.062889+0000","last_became_peered":"2024-08-17T17:05:34.062889+0000","last_unstale":"2024-08-17T17:06:10.260221+0000","last_undegraded":"2024-08-17T17:06:10.260221+0000","last_fullsized":"2024-08-17T17:06:10.260221+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:55.417929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:12.094109+0000","last_change":"2024-08-17T17:04:29.328629+0000","last_active":"2024-08-17T17:06:12.094109+0000","last_peered":"2024-08-17T17:06:12.094109+0000","last_clean":"2024-08-17T17:06:12.094109+0000","last_became_active":"2024-08-17T17:04:24.349917+0000","last_became_peered":"2024-08-17T17:04:24.349917+0000","last_unstale":"2024-08-17T17:06:12.094109+0000","last_undegraded":"2024-08-17T17:06:12.094109+0000","last_fullsized":"2024-08-17T17:06:12.094109+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:53.984346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029370500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258218+0000","last_change":"2024-08-17T17:05:34.097349+0000","last_active":"2024-08-17T17:06:10.258218+0000","last_peered":"2024-08-17T17:06:10.258218+0000","last_clean":"2024-08-17T17:06:10.258218+0000","last_became_active":"2024-08-17T17:05:34.096785+0000","last_became_peered":"2024-08-17T17:05:34.096785+0000","last_unstale":"2024-08-17T17:06:10.258218+0000","last_undegraded":"2024-08-17T17:06:10.258218+0000","last_fullsized":"2024-08-17T17:06:10.258218+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:35.560324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258489+0000","last_change":"2024-08-17T17:05:34.069234+0000","last_active":"2024-08-17T17:06:10.258489+0000","last_peered":"2024-08-17T17:06:10.258489+0000","last_clean":"2024-08-17T17:06:10.258489+0000","last_became_active":"2024-08-17T17:05:34.068970+0000","last_became_peered":"2024-08-17T17:05:34.068970+0000","last_unstale":"2024-08-17T17:06:10.258489+0000","last_undegraded":"2024-08-17T17:06:10.258489+0000","last_fullsized":"2024-08-17T17:06:10.258489+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:36.618330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.864751+0000","last_change":"2024-08-17T17:04:29.455481+0000","last_active":"2024-08-17T17:06:11.864751+0000","last_peered":"2024-08-17T17:06:11.864751+0000","last_clean":"2024-08-17T17:06:11.864751+0000","last_became_active":"2024-08-17T17:04:24.350074+0000","last_became_peered":"2024-08-17T17:04:24.350074+0000","last_unstale":"2024-08-17T17:06:11.864751+0000","last_undegraded":"2024-08-17T17:06:11.864751+0000","last_fullsized":"2024-08-17T17:06:11.864751+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:30.046202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025965899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952534+0000","last_change":"2024-08-17T17:05:34.260557+0000","last_active":"2024-08-17T17:06:10.952534+0000","last_peered":"2024-08-17T17:06:10.952534+0000","last_clean":"2024-08-17T17:06:10.952534+0000","last_became_active":"2024-08-17T17:05:34.260254+0000","last_became_peered":"2024-08-17T17:05:34.260254+0000","last_unstale":"2024-08-17T17:06:10.952534+0000","last_undegraded":"2024-08-17T17:06:10.952534+0000","last_fullsized":"2024-08-17T17:06:10.952534+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:07.261500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261176+0000","last_change":"2024-08-17T17:05:34.070409+0000","last_active":"2024-08-17T17:06:10.261176+0000","last_peered":"2024-08-17T17:06:10.261176+0000","last_clean":"2024-08-17T17:06:10.261176+0000","last_became_active":"2024-08-17T17:05:34.070058+0000","last_became_peered":"2024-08-17T17:05:34.070058+0000","last_unstale":"2024-08-17T17:06:10.261176+0000","last_undegraded":"2024-08-17T17:06:10.261176+0000","last_fullsized":"2024-08-17T17:06:10.261176+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:35.963134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.259010+0000","last_change":"2024-08-17T17:04:29.328607+0000","last_active":"2024-08-17T17:06:10.259010+0000","last_peered":"2024-08-17T17:06:10.259010+0000","last_clean":"2024-08-17T17:06:10.259010+0000","last_became_active":"2024-08-17T17:04:24.349906+0000","last_became_peered":"2024-08-17T17:04:24.349906+0000","last_unstale":"2024-08-17T17:06:10.259010+0000","last_undegraded":"2024-08-17T17:06:10.259010+0000","last_fullsized":"2024-08-17T17:06:10.259010+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:29.809856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027426399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260289+0000","last_change":"2024-08-17T17:05:34.261203+0000","last_active":"2024-08-17T17:06:10.260289+0000","last_peered":"2024-08-17T17:06:10.260289+0000","last_clean":"2024-08-17T17:06:10.260289+0000","last_became_active":"2024-08-17T17:05:34.260903+0000","last_became_peered":"2024-08-17T17:05:34.260903+0000","last_unstale":"2024-08-17T17:06:10.260289+0000","last_undegraded":"2024-08-17T17:06:10.260289+0000","last_fullsized":"2024-08-17T17:06:10.260289+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:41:59.669408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.953049+0000","last_change":"2024-08-17T17:05:34.070680+0000","last_active":"2024-08-17T17:06:10.953049+0000","last_peered":"2024-08-17T17:06:10.953049+0000","last_clean":"2024-08-17T17:06:10.953049+0000","last_became_active":"2024-08-17T17:05:34.070393+0000","last_became_peered":"2024-08-17T17:05:34.070393+0000","last_unstale":"2024-08-17T17:06:10.953049+0000","last_undegraded":"2024-08-17T17:06:10.953049+0000","last_fullsized":"2024-08-17T17:06:10.953049+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:41.610448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260523+0000","last_change":"2024-08-17T17:04:29.332851+0000","last_active":"2024-08-17T17:06:10.260523+0000","last_peered":"2024-08-17T17:06:10.260523+0000","last_clean":"2024-08-17T17:06:10.260523+0000","last_became_active":"2024-08-17T17:04:24.312005+0000","last_became_peered":"2024-08-17T17:04:24.312005+0000","last_unstale":"2024-08-17T17:06:10.260523+0000","last_undegraded":"2024-08-17T17:06:10.260523+0000","last_fullsized":"2024-08-17T17:06:10.260523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:53.614800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029301899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.256775+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:06:10.256775+0000","last_peered":"2024-08-17T17:06:10.256775+0000","last_clean":"2024-08-17T17:06:10.256775+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:06:10.256775+0000","last_undegraded":"2024-08-17T17:06:10.256775+0000","last_fullsized":"2024-08-17T17:06:10.256775+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257303+0000","last_change":"2024-08-17T17:05:34.099166+0000","last_active":"2024-08-17T17:06:10.257303+0000","last_peered":"2024-08-17T17:06:10.257303+0000","last_clean":"2024-08-17T17:06:10.257303+0000","last_became_active":"2024-08-17T17:05:34.098703+0000","last_became_peered":"2024-08-17T17:05:34.098703+0000","last_unstale":"2024-08-17T17:06:10.257303+0000","last_undegraded":"2024-08-17T17:06:10.257303+0000","last_fullsized":"2024-08-17T17:06:10.257303+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:05:49.884425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257681+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:06:10.257681+0000","last_peered":"2024-08-17T17:06:10.257681+0000","last_clean":"2024-08-17T17:06:10.257681+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:06:10.257681+0000","last_undegraded":"2024-08-17T17:06:10.257681+0000","last_fullsized":"2024-08-17T17:06:10.257681+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258175+0000","last_change":"2024-08-17T17:05:34.099709+0000","last_active":"2024-08-17T17:06:10.258175+0000","last_peered":"2024-08-17T17:06:10.258175+0000","last_clean":"2024-08-17T17:06:10.258175+0000","last_became_active":"2024-08-17T17:05:34.098666+0000","last_became_peered":"2024-08-17T17:05:34.098666+0000","last_unstale":"2024-08-17T17:06:10.258175+0000","last_undegraded":"2024-08-17T17:06:10.258175+0000","last_fullsized":"2024-08-17T17:06:10.258175+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:59:12.380450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952136+0000","last_change":"2024-08-17T17:05:34.074152+0000","last_active":"2024-08-17T17:06:10.952136+0000","last_peered":"2024-08-17T17:06:10.952136+0000","last_clean":"2024-08-17T17:06:10.952136+0000","last_became_active":"2024-08-17T17:05:34.073729+0000","last_became_peered":"2024-08-17T17:05:34.073729+0000","last_unstale":"2024-08-17T17:06:10.952136+0000","last_undegraded":"2024-08-17T17:06:10.952136+0000","last_fullsized":"2024-08-17T17:06:10.952136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:09.916987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260274+0000","last_change":"2024-08-17T17:05:34.070202+0000","last_active":"2024-08-17T17:06:10.260274+0000","last_peered":"2024-08-17T17:06:10.260274+0000","last_clean":"2024-08-17T17:06:10.260274+0000","last_became_active":"2024-08-17T17:05:34.069297+0000","last_became_peered":"2024-08-17T17:05:34.069297+0000","last_unstale":"2024-08-17T17:06:10.260274+0000","last_undegraded":"2024-08-17T17:06:10.260274+0000","last_fullsized":"2024-08-17T17:06:10.260274+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:32.343433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260196+0000","last_change":"2024-08-17T17:05:34.261470+0000","last_active":"2024-08-17T17:06:10.260196+0000","last_peered":"2024-08-17T17:06:10.260196+0000","last_clean":"2024-08-17T17:06:10.260196+0000","last_became_active":"2024-08-17T17:05:34.260973+0000","last_became_peered":"2024-08-17T17:05:34.260973+0000","last_unstale":"2024-08-17T17:06:10.260196+0000","last_undegraded":"2024-08-17T17:06:10.260196+0000","last_fullsized":"2024-08-17T17:06:10.260196+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:15.201788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952820+0000","last_change":"2024-08-17T17:05:34.074865+0000","last_active":"2024-08-17T17:06:10.952820+0000","last_peered":"2024-08-17T17:06:10.952820+0000","last_clean":"2024-08-17T17:06:10.952820+0000","last_became_active":"2024-08-17T17:05:34.074361+0000","last_became_peered":"2024-08-17T17:05:34.074361+0000","last_unstale":"2024-08-17T17:06:10.952820+0000","last_undegraded":"2024-08-17T17:06:10.952820+0000","last_fullsized":"2024-08-17T17:06:10.952820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:14.357688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260189+0000","last_change":"2024-08-17T17:05:34.096604+0000","last_active":"2024-08-17T17:06:10.260189+0000","last_peered":"2024-08-17T17:06:10.260189+0000","last_clean":"2024-08-17T17:06:10.260189+0000","last_became_active":"2024-08-17T17:05:34.095282+0000","last_became_peered":"2024-08-17T17:05:34.095282+0000","last_unstale":"2024-08-17T17:06:10.260189+0000","last_undegraded":"2024-08-17T17:06:10.260189+0000","last_fullsized":"2024-08-17T17:06:10.260189+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:54.951864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260449+0000","last_change":"2024-08-17T17:05:34.099938+0000","last_active":"2024-08-17T17:06:10.260449+0000","last_peered":"2024-08-17T17:06:10.260449+0000","last_clean":"2024-08-17T17:06:10.260449+0000","last_became_active":"2024-08-17T17:05:34.099559+0000","last_became_peered":"2024-08-17T17:05:34.099559+0000","last_unstale":"2024-08-17T17:06:10.260449+0000","last_undegraded":"2024-08-17T17:06:10.260449+0000","last_fullsized":"2024-08-17T17:06:10.260449+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:20.237416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260422+0000","last_change":"2024-08-17T17:05:34.073485+0000","last_active":"2024-08-17T17:06:10.260422+0000","last_peered":"2024-08-17T17:06:10.260422+0000","last_clean":"2024-08-17T17:06:10.260422+0000","last_became_active":"2024-08-17T17:05:34.073214+0000","last_became_peered":"2024-08-17T17:05:34.073214+0000","last_unstale":"2024-08-17T17:06:10.260422+0000","last_undegraded":"2024-08-17T17:06:10.260422+0000","last_fullsized":"2024-08-17T17:06:10.260422+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:41.426938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952867+0000","last_change":"2024-08-17T17:05:34.261376+0000","last_active":"2024-08-17T17:06:10.952867+0000","last_peered":"2024-08-17T17:06:10.952867+0000","last_clean":"2024-08-17T17:06:10.952867+0000","last_became_active":"2024-08-17T17:05:34.261104+0000","last_became_peered":"2024-08-17T17:05:34.261104+0000","last_unstale":"2024-08-17T17:06:10.952867+0000","last_undegraded":"2024-08-17T17:06:10.952867+0000","last_fullsized":"2024-08-17T17:06:10.952867+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:09.996445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.258529+0000","last_change":"2024-08-17T17:05:34.067214+0000","last_active":"2024-08-17T17:06:10.258529+0000","last_peered":"2024-08-17T17:06:10.258529+0000","last_clean":"2024-08-17T17:06:10.258529+0000","last_became_active":"2024-08-17T17:05:34.066832+0000","last_became_peered":"2024-08-17T17:05:34.066832+0000","last_unstale":"2024-08-17T17:06:10.258529+0000","last_undegraded":"2024-08-17T17:06:10.258529+0000","last_fullsized":"2024-08-17T17:06:10.258529+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:03.309536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260967+0000","last_change":"2024-08-17T17:05:34.100262+0000","last_active":"2024-08-17T17:06:10.260967+0000","last_peered":"2024-08-17T17:06:10.260967+0000","last_clean":"2024-08-17T17:06:10.260967+0000","last_became_active":"2024-08-17T17:05:34.099984+0000","last_became_peered":"2024-08-17T17:05:34.099984+0000","last_unstale":"2024-08-17T17:06:10.260967+0000","last_undegraded":"2024-08-17T17:06:10.260967+0000","last_fullsized":"2024-08-17T17:06:10.260967+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:31.894964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257119+0000","last_change":"2024-08-17T17:05:34.070037+0000","last_active":"2024-08-17T17:06:10.257119+0000","last_peered":"2024-08-17T17:06:10.257119+0000","last_clean":"2024-08-17T17:06:10.257119+0000","last_became_active":"2024-08-17T17:05:34.069229+0000","last_became_peered":"2024-08-17T17:05:34.069229+0000","last_unstale":"2024-08-17T17:06:10.257119+0000","last_undegraded":"2024-08-17T17:06:10.257119+0000","last_fullsized":"2024-08-17T17:06:10.257119+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:49:22.627947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257256+0000","last_change":"2024-08-17T17:05:34.096779+0000","last_active":"2024-08-17T17:06:10.257256+0000","last_peered":"2024-08-17T17:06:10.257256+0000","last_clean":"2024-08-17T17:06:10.257256+0000","last_became_active":"2024-08-17T17:05:34.096470+0000","last_became_peered":"2024-08-17T17:05:34.096470+0000","last_unstale":"2024-08-17T17:06:10.257256+0000","last_undegraded":"2024-08-17T17:06:10.257256+0000","last_fullsized":"2024-08-17T17:06:10.257256+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:00.344836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952696+0000","last_change":"2024-08-17T17:05:34.071766+0000","last_active":"2024-08-17T17:06:10.952696+0000","last_peered":"2024-08-17T17:06:10.952696+0000","last_clean":"2024-08-17T17:06:10.952696+0000","last_became_active":"2024-08-17T17:05:34.071377+0000","last_became_peered":"2024-08-17T17:05:34.071377+0000","last_unstale":"2024-08-17T17:06:10.952696+0000","last_undegraded":"2024-08-17T17:06:10.952696+0000","last_fullsized":"2024-08-17T17:06:10.952696+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:49.697261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257221+0000","last_change":"2024-08-17T17:05:34.099471+0000","last_active":"2024-08-17T17:06:10.257221+0000","last_peered":"2024-08-17T17:06:10.257221+0000","last_clean":"2024-08-17T17:06:10.257221+0000","last_became_active":"2024-08-17T17:05:34.099159+0000","last_became_peered":"2024-08-17T17:05:34.099159+0000","last_unstale":"2024-08-17T17:06:10.257221+0000","last_undegraded":"2024-08-17T17:06:10.257221+0000","last_fullsized":"2024-08-17T17:06:10.257221+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:30.799877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260180+0000","last_change":"2024-08-17T17:05:34.067217+0000","last_active":"2024-08-17T17:06:10.260180+0000","last_peered":"2024-08-17T17:06:10.260180+0000","last_clean":"2024-08-17T17:06:10.260180+0000","last_became_active":"2024-08-17T17:05:34.066933+0000","last_became_peered":"2024-08-17T17:05:34.066933+0000","last_unstale":"2024-08-17T17:06:10.260180+0000","last_undegraded":"2024-08-17T17:06:10.260180+0000","last_fullsized":"2024-08-17T17:06:10.260180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:56.648864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.257887+0000","last_change":"2024-08-17T17:05:34.097611+0000","last_active":"2024-08-17T17:06:10.257887+0000","last_peered":"2024-08-17T17:06:10.257887+0000","last_clean":"2024-08-17T17:06:10.257887+0000","last_became_active":"2024-08-17T17:05:34.097322+0000","last_became_peered":"2024-08-17T17:05:34.097322+0000","last_unstale":"2024-08-17T17:06:10.257887+0000","last_undegraded":"2024-08-17T17:06:10.257887+0000","last_fullsized":"2024-08-17T17:06:10.257887+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:36:45.311214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260125+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:06:10.260125+0000","last_peered":"2024-08-17T17:06:10.260125+0000","last_clean":"2024-08-17T17:06:10.260125+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:06:10.260125+0000","last_undegraded":"2024-08-17T17:06:10.260125+0000","last_fullsized":"2024-08-17T17:06:10.260125+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.260476+0000","last_change":"2024-08-17T17:05:34.101996+0000","last_active":"2024-08-17T17:06:10.260476+0000","last_peered":"2024-08-17T17:06:10.260476+0000","last_clean":"2024-08-17T17:06:10.260476+0000","last_became_active":"2024-08-17T17:05:34.101095+0000","last_became_peered":"2024-08-17T17:05:34.101095+0000","last_unstale":"2024-08-17T17:06:10.260476+0000","last_undegraded":"2024-08-17T17:06:10.260476+0000","last_fullsized":"2024-08-17T17:06:10.260476+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:42.003093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.261087+0000","last_change":"2024-08-17T17:05:34.069289+0000","last_active":"2024-08-17T17:06:10.261087+0000","last_peered":"2024-08-17T17:06:10.261087+0000","last_clean":"2024-08-17T17:06:10.261087+0000","last_became_active":"2024-08-17T17:05:34.068969+0000","last_became_peered":"2024-08-17T17:05:34.068969+0000","last_unstale":"2024-08-17T17:06:10.261087+0000","last_undegraded":"2024-08-17T17:06:10.261087+0000","last_fullsized":"2024-08-17T17:06:10.261087+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:07:29.328313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:06:11.297438+0000","last_change":"2024-08-17T17:06:11.297438+0000","last_active":"2024-08-17T17:06:11.297438+0000","last_peered":"2024-08-17T17:06:11.297438+0000","last_clean":"2024-08-17T17:06:11.297438+0000","last_became_active":"2024-08-17T17:06:11.297134+0000","last_became_peered":"2024-08-17T17:06:11.297134+0000","last_unstale":"2024-08-17T17:06:11.297438+0000","last_undegraded":"2024-08-17T17:06:11.297438+0000","last_fullsized":"2024-08-17T17:06:11.297438+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:51.294632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:06:10.952797+0000","last_change":"2024-08-17T17:05:34.066388+0000","last_active":"2024-08-17T17:06:10.952797+0000","last_peered":"2024-08-17T17:06:10.952797+0000","last_clean":"2024-08-17T17:06:10.952797+0000","last_became_active":"2024-08-17T17:05:34.066076+0000","last_became_peered":"2024-08-17T17:05:34.066076+0000","last_unstale":"2024-08-17T17:06:10.952797+0000","last_undegraded":"2024-08-17T17:06:10.952797+0000","last_fullsized":"2024-08-17T17:06:10.952797+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:04.886951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1372160,"data_stored":921386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.93799999999999994,"15min":1.0029999999999999},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71199999999999997,"5min":13.132999999999999,"15min":13.132999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.41999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71899999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.40100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75700000000000001,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":1.028,"15min":1.028},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66700000000000004,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.135,"15min":1.2290000000000001},"min":{"1min":0.42399999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70099999999999996,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.44700000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69299999999999995,"5min":0.878,"15min":0.878},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.42499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73199999999999998,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.67500000000000004}]},{"osd":4,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66100000000000003,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.48699999999999999,"15min":0.49299999999999999},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.61599999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.315,"5min":0.309,"15min":0.309},"max":{"1min":0.61099999999999999,"5min":1.044,"15min":1.044},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66200000000000003,"5min":1.091,"15min":1.091},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.69499999999999995,"5min":1.117,"15min":1.117},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.52300000000000002,"15min":0.52800000000000002},"min":{"1min":0.373,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68899999999999995,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.624}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":920994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.52600000000000002,"15min":0.51900000000000002},"min":{"1min":0.45500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.68700000000000006,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.94299999999999995,"15min":0.67000000000000004},"min":{"1min":0.45600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.75700000000000001,"5min":19.114999999999998,"15min":19.114999999999998},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.437,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.495},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.41699999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.82699999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.44}]},{"osd":2,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.40799999999999997,"5min":0.315,"15min":0.315},"max":{"1min":0.79900000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.434},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.53600000000000003,"15min":0.52600000000000002},"min":{"1min":0.42499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.48099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":0.78300000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.55300000000000005,"15min":0.54500000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.40899999999999997,"15min":0.39300000000000002},"min":{"1min":0.28199999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.437,"15min":0.41699999999999998},"min":{"1min":0.39500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.439,"15min":0.42099999999999999},"min":{"1min":0.29599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.47799999999999998,"15min":0.45900000000000002},"min":{"1min":0.29799999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.754,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:05:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.497,"15min":0.49099999999999999},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.49399999999999999,"15min":0.48999999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41999999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717388,"statfs":{"total":95995035648,"available":95966605312,"internally_reserved":0,"allocated":761856,"data_stored":329909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.377,"5min":0.308,"15min":0.308},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.52100000000000002,"15min":0.52200000000000002},"min":{"1min":0.35399999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.68600000000000005,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.44}]},{"osd":1,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71699999999999997,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.50600000000000001,"15min":0.496},"min":{"1min":0.36699999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74299999999999999,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.376,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75700000000000001,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.40999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81499999999999995,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.434,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73299999999999998,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.503},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.38500000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.79600000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.377}]},{"osd":4,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.373},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.59199999999999997,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.41399999999999998,"15min":0.39100000000000001},"min":{"1min":0.312,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65400000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42499999999999999,"15min":0.40300000000000002},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63200000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46400000000000002,"15min":0.44},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.63,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:05:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.46600000000000003,"15min":0.46200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.38800000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":330956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.073,"15min":1.323},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69799999999999995,"5min":16.233000000000001,"15min":16.233000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.51500000000000001,"15min":0.52300000000000002},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.65900000000000003,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.60599999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.056,"15min":1.306},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67600000000000005,"5min":16.245000000000001,"15min":16.245000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.55500000000000005,"15min":0.57299999999999995},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.624}]},{"osd":2,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69299999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.498},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.69499999999999995,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.65600000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53600000000000003,"15min":0.54600000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.0600000000000001,"15min":1.3},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.63800000000000001,"5min":16.27,"15min":16.27},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.40799999999999997,"15min":0.39000000000000001},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.433}]},{"osd":6,"last update":"Sat Aug 17 17:06:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.40400000000000003,"15min":0.38700000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.57899999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42199999999999999,"15min":0.40200000000000002},"min":{"1min":0.36199999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.35899999999999999}]},{"osd":7,"last update":"Sat Aug 17 17:05:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.46500000000000002,"15min":0.45500000000000002},"min":{"1min":0.38900000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.44800000000000001,"15min":0.432},"min":{"1min":0.34499999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.45000000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":753664,"data_stored":329336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.376,"15min":0.38900000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.5},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40100000000000002,"15min":0.41499999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61799999999999999,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41499999999999998,"15min":0.436},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.67700000000000005,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39100000000000001,"15min":0.41099999999999998},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.43099999999999999,"15min":0.45100000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54100000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41899999999999998,"15min":0.435},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60799999999999998,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56599999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58099999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.82899999999999996,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77500000000000002,"5min":0.877,"15min":0.877},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58999999999999997,"15min":0.59899999999999998},"min":{"1min":0.45800000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:06:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76800000000000002,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.621,"5min":0.61299999999999999,"15min":0.61099999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.80200000000000005,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Aug 17 17:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.59499999999999997,"15min":0.59399999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.45800000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.68899999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27760,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717392,"statfs":{"total":95995035648,"available":95966609408,"internally_reserved":0,"allocated":757760,"data_stored":329862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.34000000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.36399999999999999,"15min":0.34999999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.436}]},{"osd":1,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35299999999999998,"15min":0.34100000000000003},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51700000000000002,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Aug 17 17:06:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.44,"15min":0.44},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.504,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.40899999999999997,"15min":0.42299999999999999},"min":{"1min":0.29699999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.496,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Aug 17 17:05:35 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68999999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58799999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.83699999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.67900000000000005}]},{"osd":5,"last update":"Sat Aug 17 17:05:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59499999999999997,"15min":0.60399999999999998},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.74299999999999999,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.71499999999999997}]},{"osd":6,"last update":"Sat Aug 17 17:05:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77600000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Aug 17 17:05:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.437,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78900000000000003,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.623,"5min":0.63500000000000001,"15min":0.63700000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.81299999999999994}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":328992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:06:31 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.376,"15min":0.32800000000000001},"min":{"1min":0.255,"5min":0.155,"15min":0.155},"max":{"1min":0.746,"5min":1.02,"15min":1.02},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.379,"15min":0.32800000000000001},"min":{"1min":0.24399999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.53700000000000003,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:05:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.376},"min":{"1min":0.255,"5min":0.245,"15min":0.245},"max":{"1min":0.57299999999999995,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.38},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.189},"max":{"1min":0.54400000000000004,"5min":1.054,"15min":1.054},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:06:26 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.438},"min":{"1min":0.32300000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61299999999999999,"5min":1.002,"15min":1.002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40799999999999997,"15min":0.40500000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.626,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:05:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57599999999999996,"15min":0.56200000000000006},"min":{"1min":0.36499999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.157,"5min":1.157,"15min":1.157},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59599999999999997,"15min":0.59199999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.85999999999999999}]},{"osd":5,"last update":"Sat Aug 17 17:06:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.35799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.81999999999999995,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78700000000000003,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.73699999999999999}]},{"osd":6,"last update":"Sat Aug 17 17:06:26 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.63800000000000001,"15min":0.66000000000000003},"min":{"1min":0.45900000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73799999999999999,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.45100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.746,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.61599999999999999}]},{"osd":7,"last update":"Sat Aug 17 17:05:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42899999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.78400000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.41499999999999998,"5min":0.375,"15min":0.375},"max":{"1min":0.85099999999999998,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.753}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32440,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712712,"statfs":{"total":95995035648,"available":95961817088,"internally_reserved":0,"allocated":1355776,"data_stored":920491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:05:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.375,"15min":0.40300000000000002},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":0.46600000000000003,"5min":2.0739999999999998,"15min":2.0739999999999998},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57099999999999995,"5min":2.008,"15min":2.008},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:05:53 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.42899999999999999,"15min":0.46400000000000002},"min":{"1min":0.26300000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.53500000000000003,"5min":2.1429999999999998,"15min":2.1429999999999998},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.48499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:06:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.442,"15min":0.45200000000000001},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.57999999999999996,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.432,"15min":0.439},"min":{"1min":0.26900000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66600000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:05:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.80600000000000005,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53500000000000003},"min":{"1min":0.38100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Aug 17 17:06:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.626},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.52700000000000002,"15min":0.497},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76100000000000001,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:06:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.41399999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60799999999999998,"15min":0.58999999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.69699999999999995}]},{"osd":7,"last update":"Sat Aug 17 17:05:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:06:33.985 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:06:33.985 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:06:33.985 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:06:33.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:06:34.198 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:35.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:35 smithi064 ceph-mon[30574]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:35.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:35 smithi064 ceph-mon[30574]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:35.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:35 smithi064 ceph-mon[40964]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:35.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:35 smithi064 ceph-mon[40964]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:35 smithi175 ceph-mon[34606]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:06:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:35 smithi175 ceph-mon[34606]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:36.049 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:06:36.049 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:06:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:36 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/3047752763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:06:36.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:36 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/3047752763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:06:36.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:36 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/3047752763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:06:36.706 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:06:36.706 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:06:36.715 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-08-17T17:06:36.716 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:06:36.734 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:06:36.735 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:06:36.736 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T17:06:36.741 INFO:tasks.workunit:Pulling workunits from ref reef 2024-08-17T17:06:36.741 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T17:06:36.742 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:06:36.742 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:06:36.742 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-08-17T17:06:36.743 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2024-08-17T17:06:36.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-08-17T17:06:36.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:06:36.761 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T17:06:36.761 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:06:36.817 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T17:06:36.818 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T17:06:36.873 INFO:tasks.workunit:timeout=3h 2024-08-17T17:06:36.873 INFO:tasks.workunit:cleanup=True 2024-08-17T17:06:36.874 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:06:36.931 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T17:06:36.959 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:37 smithi064 ceph-mon[30574]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:37.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:37 smithi064 ceph-mon[40964]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:37 smithi175 ceph-mon[34606]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:39.170 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:38 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:38] ENGINE Bus STOPPING 2024-08-17T17:06:39.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: from='client.? ' entity='client.admin' 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: from='client.? ' entity='client.admin' 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:06:39.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:39] ENGINE Bus STOPPED 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:39] ENGINE Bus STARTING 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:39] ENGINE Serving on http://:::9283 2024-08-17T17:06:39.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:06:39] ENGINE Bus STARTED 2024-08-17T17:06:39.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-17T17:06:39.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: from='client.? ' entity='client.admin' 2024-08-17T17:06:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:06:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:06:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:06:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:39 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:06:39.836 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:41.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:41 smithi064 ceph-mon[30574]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:41.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:41 smithi064 ceph-mon[40964]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:41 smithi175 ceph-mon[34606]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:42.028 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:41] "GET /metrics HTTP/1.1" 200 33924 "" "Prometheus/2.43.0" 2024-08-17T17:06:42.286 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T17:06:42.332 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T17:06:42.334 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:06:42.334 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T17:06:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:42 smithi175 ceph-mon[34606]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:42 smithi064 ceph-mon[30574]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:42 smithi064 ceph-mon[40964]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:42.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-08-17T17:06:43.093 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:44.021 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T17:06:44.138 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T17:06:44.186 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:06:44.188 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:06:44.188 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T17:06:44.230 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:06:44.234 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:06:44.234 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T17:06:44.251 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-08-17T17:06:44.252 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-08-17T17:06:44.252 DEBUG:teuthology.orchestra.run.smithi064:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-08-17T17:06:44.347 INFO:tasks.workunit.client.0.smithi064.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-08-17T17:06:44.347 INFO:tasks.workunit.client.0.smithi064.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-08-17T17:06:44.347 INFO:tasks.workunit.client.0.smithi064.stderr:+ TMPDIR=/tmp/rbd_import_export_80524 2024-08-17T17:06:44.347 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm -rf /tmp/rbd_import_export_80524 2024-08-17T17:06:44.348 INFO:tasks.workunit.client.0.smithi064.stderr:+ mkdir /tmp/rbd_import_export_80524 2024-08-17T17:06:44.349 INFO:tasks.workunit.client.0.smithi064.stderr:+ trap 'rm -rf /tmp/rbd_import_export_80524' INT TERM EXIT 2024-08-17T17:06:44.349 INFO:tasks.workunit.client.0.smithi064.stderr:+ mkdir foo.80524 2024-08-17T17:06:44.349 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import foo.80524 foo.dir 2024-08-17T17:06:44.434 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: cannot import a directory 2024-08-17T17:06:44.435 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 0% complete...failed. 2024-08-17T17:06:44.435 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: import failed: (21) Is a directory 2024-08-17T17:06:44.454 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:06:44.454 INFO:tasks.workunit.client.0.smithi064.stderr:+ rmdir foo.80524 2024-08-17T17:06:44.487 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_80524/img bs=1k count=1 seek=10 2024-08-17T17:06:44.488 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2024-08-17T17:06:44.488 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2024-08-17T17:06:44.488 INFO:tasks.workunit.client.0.smithi064.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.732e-05 s, 21.6 MB/s 2024-08-17T17:06:44.488 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_80524/img bs=1k count=10 seek=100 2024-08-17T17:06:44.489 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records in 2024-08-17T17:06:44.489 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records out 2024-08-17T17:06:44.489 INFO:tasks.workunit.client.0.smithi064.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.9851e-05 s, 171 MB/s 2024-08-17T17:06:44.489 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_80524/img bs=1k count=100 seek=1000 2024-08-17T17:06:44.490 INFO:tasks.workunit.client.0.smithi064.stderr:60+1 records in 2024-08-17T17:06:44.490 INFO:tasks.workunit.client.0.smithi064.stderr:60+1 records out 2024-08-17T17:06:44.490 INFO:tasks.workunit.client.0.smithi064.stderr:61656 bytes (62 kB, 60 KiB) copied, 0.00018126 s, 340 MB/s 2024-08-17T17:06:44.490 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_80524/img bs=1k seek=10000 2024-08-17T17:06:44.491 INFO:tasks.workunit.client.0.smithi064.stderr:137+1 records in 2024-08-17T17:06:44.491 INFO:tasks.workunit.client.0.smithi064.stderr:137+1 records out 2024-08-17T17:06:44.491 INFO:tasks.workunit.client.0.smithi064.stderr:140952 bytes (141 kB, 138 KiB) copied, 0.000410162 s, 344 MB/s 2024-08-17T17:06:44.491 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_80524/img bs=1k seek=100000 2024-08-17T17:06:44.492 INFO:tasks.workunit.client.0.smithi064.stderr:60+1 records in 2024-08-17T17:06:44.492 INFO:tasks.workunit.client.0.smithi064.stderr:60+1 records out 2024-08-17T17:06:44.492 INFO:tasks.workunit.client.0.smithi064.stderr:61608 bytes (62 kB, 60 KiB) copied, 0.000178068 s, 346 MB/s 2024-08-17T17:06:44.492 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_80524/img bs=1k seek=1000000 2024-08-17T17:06:44.493 INFO:tasks.workunit.client.0.smithi064.stderr:154+1 records in 2024-08-17T17:06:44.493 INFO:tasks.workunit.client.0.smithi064.stderr:154+1 records out 2024-08-17T17:06:44.493 INFO:tasks.workunit.client.0.smithi064.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.00040721 s, 388 MB/s 2024-08-17T17:06:44.493 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2024-08-17T17:06:44.535 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2024-08-17T17:06:44.535 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2024-08-17T17:06:44.538 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:06:44.539 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:45.065 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-08-17T17:06:45.065 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:06:45.068 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg /tmp/rbd_import_export_80524/img2 2024-08-17T17:06:45.099 INFO:teuthology.orchestra.run.smithi064.stdout:wait for mgr daemons to upgrade 2024-08-17T17:06:45.269 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:45.274 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg - 2024-08-17T17:06:45.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:45 smithi064 ceph-mon[30574]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:45.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:45 smithi064 ceph-mon[40964]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:45 smithi175 ceph-mon[34606]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:06:45.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-08-17T17:06:45.959 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:45.963 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2024-08-17T17:06:46.017 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:46.061 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-17T17:06:46.064 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img2 2024-08-17T17:06:46.698 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img3 2024-08-17T17:06:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:46 smithi175 ceph-mon[34606]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:47.168 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img2 /tmp/rbd_import_export_80524/img3 2024-08-17T17:06:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:46 smithi064 ceph-mon[30574]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:46 smithi064 ceph-mon[40964]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:47.346 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format - testimg 2024-08-17T17:06:47.713 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:47.713 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:06:47.716 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg /tmp/rbd_import_export_80524/img2 2024-08-17T17:06:47.894 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:47.898 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg - 2024-08-17T17:06:48.570 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:48.575 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2024-08-17T17:06:48.662 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-17T17:06:48.665 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img2 2024-08-17T17:06:49.332 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img3 2024-08-17T17:06:49.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:49 smithi175 ceph-mon[34606]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:49 smithi175 ceph-mon[34606]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:49 smithi064 ceph-mon[30574]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:49 smithi064 ceph-mon[30574]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:49.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:49 smithi064 ceph-mon[40964]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:49 smithi064 ceph-mon[40964]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:49.843 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img2 /tmp/rbd_import_export_80524/img3 2024-08-17T17:06:50.074 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd help export 2024-08-17T17:06:50.074 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -q export-format 2024-08-17T17:06:50.097 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_80524/img bs=1k count=10 seek=100 2024-08-17T17:06:50.098 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records in 2024-08-17T17:06:50.098 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records out 2024-08-17T17:06:50.098 INFO:tasks.workunit.client.0.smithi064.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.1538e-05 s, 143 MB/s 2024-08-17T17:06:50.098 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:50.161 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:50.162 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:06:50.164 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap 2024-08-17T17:06:50.279 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-17T17:06:50.286 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd image-meta set testimg key1 value1 2024-08-17T17:06:50.345 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd image-meta list testimg 2024-08-17T17:06:50.390 INFO:tasks.workunit.client.0.smithi064.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-08-17T17:06:50.391 INFO:tasks.workunit.client.0.smithi064.stderr: 2024-08-17T17:06:50.391 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2024-08-17T17:06:50.391 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' 2024-08-17T17:06:50.391 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:50.436 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:50.440 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80524/img_v2 testimg_import 2024-08-17T17:06:51.292 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-17T17:06:51.295 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2024-08-17T17:06:51.334 INFO:tasks.workunit.client.0.smithi064.stdout:rbd image 'testimg_import': 2024-08-17T17:06:51.334 INFO:tasks.workunit.client.0.smithi064.stdout: size 110 KiB in 1 objects 2024-08-17T17:06:51.334 INFO:tasks.workunit.client.0.smithi064.stdout: order 22 (4 MiB objects) 2024-08-17T17:06:51.334 INFO:tasks.workunit.client.0.smithi064.stdout: snapshot_count: 1 2024-08-17T17:06:51.334 INFO:tasks.workunit.client.0.smithi064.stdout: id: 60e31ed63ce3 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: block_name_prefix: rbd_data.60e31ed63ce3 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: format: 2 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: op_features: 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: flags: 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: create_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: access_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.335 INFO:tasks.workunit.client.0.smithi064.stdout: modify_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.337 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout:rbd image 'testimg_import': 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: size 110 KiB in 1 objects 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: order 22 (4 MiB objects) 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: snapshot_count: 1 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: id: 60e31ed63ce3 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: block_name_prefix: rbd_data.60e31ed63ce3 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: format: 2 2024-08-17T17:06:51.379 INFO:tasks.workunit.client.0.smithi064.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: op_features: 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: flags: 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: create_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: access_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: modify_timestamp: Sat Aug 17 17:06:50 2024 2024-08-17T17:06:51.380 INFO:tasks.workunit.client.0.smithi064.stdout: protected: False 2024-08-17T17:06:51.382 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd image-meta list testimg_import 2024-08-17T17:06:51.428 INFO:tasks.workunit.client.0.smithi064.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-08-17T17:06:51.428 INFO:tasks.workunit.client.0.smithi064.stderr: 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 'There is 1 metadatum on this image: 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr: 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr: 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' ']' 2024-08-17T17:06:51.429 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg_import /tmp/rbd_import_export_80524/img_import 2024-08-17T17:06:51.471 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2024-08-17T17:06:51.475 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img_import 2024-08-17T17:06:51.475 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/img /tmp/rbd_import_export_80524/img_import 2024-08-17T17:06:51.476 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/img --format %b 2024-08-17T17:06:51.477 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=24 2024-08-17T17:06:51.477 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/img_import --format %b 2024-08-17T17:06:51.478 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=224 2024-08-17T17:06:51.478 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=200 2024-08-17T17:06:51.479 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=200 2024-08-17T17:06:51.479 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-08-17T17:06:51.479 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_80524/img_snap 2024-08-17T17:06:51.518 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2024-08-17T17:06:51.522 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_80524/img_snap_import 2024-08-17T17:06:51.565 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2024-08-17T17:06:51.568 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/img_snap /tmp/rbd_import_export_80524/img_snap_import 2024-08-17T17:06:51.568 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/img_snap /tmp/rbd_import_export_80524/img_snap_import 2024-08-17T17:06:51.569 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/img_snap --format %b 2024-08-17T17:06:51.570 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=224 2024-08-17T17:06:51.571 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/img_snap_import --format %b 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=224 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:51.572 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_import 2024-08-17T17:06:51.573 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_snap 2024-08-17T17:06:51.574 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_snap_import 2024-08-17T17:06:51.575 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap rm testimg_import@snap 2024-08-17T17:06:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:51 smithi175 ceph-mon[34606]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:51 smithi175 ceph-mon[34606]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:06:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:51 smithi064 ceph-mon[30574]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:51 smithi064 ceph-mon[30574]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:06:51.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:51 smithi064 ceph-mon[40964]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-08-17T17:06:51.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:51 smithi064 ceph-mon[40964]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:06:52.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:06:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:06:51] "GET /metrics HTTP/1.1" 200 33924 "" "Prometheus/2.43.0" 2024-08-17T17:06:52.291 INFO:tasks.workunit.client.0.smithi064.stderr: Removing snap: 100% complete...done. 2024-08-17T17:06:52.298 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2024-08-17T17:06:52.393 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:52.396 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap rm testimg@snap 2024-08-17T17:06:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:52 smithi175 ceph-mon[34606]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:06:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:52 smithi064 ceph-mon[30574]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:06:52.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:52 smithi064 ceph-mon[40964]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:06:53.292 INFO:tasks.workunit.client.0.smithi064.stderr: Removing snap: 100% complete...done. 2024-08-17T17:06:53.299 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2024-08-17T17:06:53.394 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:53.398 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --order 20 /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:53.458 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:53.458 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:06:53.461 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:53.507 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:53.510 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80524/img_v2 testimg_import 2024-08-17T17:06:53.578 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-17T17:06:53.582 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2024-08-17T17:06:53.583 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep order 2024-08-17T17:06:53.583 INFO:tasks.workunit.client.0.smithi064.stderr:+ awk '{print $2}' 2024-08-17T17:06:53.583 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 20 2024-08-17T17:06:53.623 INFO:tasks.workunit.client.0.smithi064.stdout:20 2024-08-17T17:06:53.623 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:53.624 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2024-08-17T17:06:53.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:53 smithi175 ceph-mon[34606]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 122 KiB/s wr, 142 op/s 2024-08-17T17:06:53.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:53 smithi175 ceph-mon[34606]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:06:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:53 smithi064 ceph-mon[30574]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 122 KiB/s wr, 142 op/s 2024-08-17T17:06:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:53 smithi064 ceph-mon[30574]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:06:53.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:53 smithi064 ceph-mon[40964]: pgmap v99: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 122 KiB/s wr, 142 op/s 2024-08-17T17:06:53.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:53 smithi064 ceph-mon[40964]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:06:53.714 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:53.717 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2024-08-17T17:06:53.807 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:53.810 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:53.862 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:53.866 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info testimg 2024-08-17T17:06:53.866 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep features 2024-08-17T17:06:53.911 INFO:tasks.workunit.client.0.smithi064.stderr:+ FEATURES_BEFORE=' features: layering 2024-08-17T17:06:53.911 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' 2024-08-17T17:06:53.911 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:53.958 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:53.962 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80524/img_v2 testimg_import 2024-08-17T17:06:54.019 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-17T17:06:54.022 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info testimg_import 2024-08-17T17:06:54.022 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep features 2024-08-17T17:06:54.065 INFO:tasks.workunit.client.0.smithi064.stderr:+ FEATURES_AFTER=' features: layering 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' ' features: layering 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' '!=' ' features: layering 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' ']' 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:54.066 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2024-08-17T17:06:54.142 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:54.145 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2024-08-17T17:06:54.219 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:06:54.222 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:54.288 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:54.293 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:54.336 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:54.340 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80524/img_v2 testimg_import 2024-08-17T17:06:54.406 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-17T17:06:54.410 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2024-08-17T17:06:54.410 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'stripe unit' 2024-08-17T17:06:54.410 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(4 KiB|4096)' 2024-08-17T17:06:54.451 INFO:tasks.workunit.client.0.smithi064.stdout: stripe unit: 4 KiB 2024-08-17T17:06:54.452 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2024-08-17T17:06:54.452 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'stripe count' 2024-08-17T17:06:54.452 INFO:tasks.workunit.client.0.smithi064.stderr:+ awk '{print $3}' 2024-08-17T17:06:54.452 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 1000 2024-08-17T17:06:54.496 INFO:tasks.workunit.client.0.smithi064.stdout:1000 2024-08-17T17:06:54.496 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/img_v2 2024-08-17T17:06:54.497 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2024-08-17T17:06:54.587 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-17T17:06:54.590 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2024-08-17T17:06:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:54 smithi175 ceph-mon[34606]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:06:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:54 smithi175 ceph-mon[34606]: pgmap v102: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-08-17T17:06:54.683 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-17T17:06:54.686 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_80524/img testimg 2024-08-17T17:06:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:54 smithi064 ceph-mon[30574]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:06:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:54 smithi064 ceph-mon[30574]: pgmap v102: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-08-17T17:06:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:54 smithi064 ceph-mon[40964]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:06:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:54 smithi064 ceph-mon[40964]: pgmap v102: 73 pgs: 73 active+clean; 799 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 129 KiB/s wr, 208 op/s 2024-08-17T17:06:54.747 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:06:54.750 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap1 2024-08-17T17:06:55.304 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-17T17:06:55.311 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap2 2024-08-17T17:06:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:55 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:55 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:55.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:55 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:06:56.308 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-17T17:06:56.315 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap protect testimg@snap2 2024-08-17T17:06:56.366 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80524/snap_protect 2024-08-17T17:06:56.414 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-17T17:06:56.417 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80524/snap_protect testimg_import 2024-08-17T17:06:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:56 smithi175 ceph-mon[34606]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:06:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:56 smithi175 ceph-mon[34606]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 366 KiB/s rd, 38 KiB/s wr, 481 op/s 2024-08-17T17:06:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:56 smithi064 ceph-mon[30574]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:06:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:56 smithi064 ceph-mon[30574]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 366 KiB/s rd, 38 KiB/s wr, 481 op/s 2024-08-17T17:06:56.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:56 smithi064 ceph-mon[40964]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:06:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:56 smithi064 ceph-mon[40964]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 366 KiB/s rd, 38 KiB/s wr, 481 op/s 2024-08-17T17:06:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:57 smithi175 ceph-mon[34606]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:06:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:57 smithi064 ceph-mon[30574]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:06:57.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:57 smithi064 ceph-mon[40964]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:06:58.335 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-17T17:06:58.339 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap1 2024-08-17T17:06:58.339 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'protected: False' 2024-08-17T17:06:58.386 INFO:tasks.workunit.client.0.smithi064.stdout: protected: False 2024-08-17T17:06:58.386 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap2 2024-08-17T17:06:58.386 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'protected: True' 2024-08-17T17:06:58.430 INFO:tasks.workunit.client.0.smithi064.stdout: protected: True 2024-08-17T17:06:58.430 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/snap_protect 2024-08-17T17:06:58.430 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap unprotect testimg@snap2 2024-08-17T17:06:58.485 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap unprotect testimg_import@snap2 2024-08-17T17:06:58.534 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap purge testimg 2024-08-17T17:06:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:58 smithi175 ceph-mon[34606]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:06:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:58 smithi175 ceph-mon[34606]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 367 KiB/s rd, 38 KiB/s wr, 482 op/s 2024-08-17T17:06:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:58 smithi064 ceph-mon[30574]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:06:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:58 smithi064 ceph-mon[30574]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 367 KiB/s rd, 38 KiB/s wr, 482 op/s 2024-08-17T17:06:58.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:58 smithi064 ceph-mon[40964]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:06:58.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:58 smithi064 ceph-mon[40964]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 367 KiB/s rd, 38 KiB/s wr, 482 op/s 2024-08-17T17:06:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:06:59 smithi175 ceph-mon[34606]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:06:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:06:59 smithi064 ceph-mon[30574]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:06:59.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:06:59 smithi064 ceph-mon[40964]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:07:00.333 INFO:tasks.workunit.client.0.smithi064.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-08-17T17:07:00.341 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap purge testimg_import 2024-08-17T17:07:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:00 smithi175 ceph-mon[34606]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:07:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:00 smithi175 ceph-mon[34606]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:00 smithi064 ceph-mon[30574]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:07:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:00 smithi064 ceph-mon[30574]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:00.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:00 smithi064 ceph-mon[40964]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:07:00.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:00 smithi064 ceph-mon[40964]: pgmap v110: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:01 smithi175 ceph-mon[34606]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:07:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:01 smithi175 ceph-mon[34606]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:07:01.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:01 smithi064 ceph-mon[40964]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:07:01.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:01 smithi064 ceph-mon[40964]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:07:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:01 smithi064 ceph-mon[30574]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:07:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:01 smithi064 ceph-mon[30574]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:07:01.761 INFO:tasks.workunit.client.0.smithi064.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-08-17T17:07:01.767 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2024-08-17T17:07:01.864 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:07:01.867 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2024-08-17T17:07:01.960 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:07:01.963 INFO:tasks.workunit.client.0.smithi064.stderr:+ tiered=0 2024-08-17T17:07:01.963 INFO:tasks.workunit.client.0.smithi064.stderr:+ ceph osd dump 2024-08-17T17:07:01.963 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep '^pool' 2024-08-17T17:07:01.964 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep ''\''rbd'\''' 2024-08-17T17:07:01.964 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep tier 2024-08-17T17:07:02.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:01] "GET /metrics HTTP/1.1" 200 33947 "" "Prometheus/2.43.0" 2024-08-17T17:07:02.394 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_80524/sparse1 2024-08-17T17:07:02.398 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2024-08-17T17:07:02.398 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2024-08-17T17:07:02.398 INFO:tasks.workunit.client.0.smithi064.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.003329 s, 315 MB/s 2024-08-17T17:07:02.398 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_80524/sparse2 2024-08-17T17:07:02.402 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2024-08-17T17:07:02.402 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2024-08-17T17:07:02.402 INFO:tasks.workunit.client.0.smithi064.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0032035 s, 327 MB/s 2024-08-17T17:07:02.402 INFO:tasks.workunit.client.0.smithi064.stderr:+ truncate /tmp/rbd_import_export_80524/sparse2 -s 2M 2024-08-17T17:07:02.413 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2024-08-17T17:07:02.461 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2024-08-17T17:07:02.462 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2024-08-17T17:07:02.464 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:07:02.465 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80524/sparse1 2024-08-17T17:07:02.531 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-08-17T17:07:02.532 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:02.532 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:07:02.535 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2024-08-17T17:07:02.535 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse1 2024-08-17T17:07:02.535 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-08-17T17:07:02.582 INFO:tasks.workunit.client.0.smithi064.stdout:sparse1 2 MiB 2 2024-08-17T17:07:02.583 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse1 2024-08-17T17:07:02.583 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse1 2024-08-17T17:07:02.583 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse1 2024-08-17T17:07:02.583 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:02.583 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:02.627 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.6109fc6c8d91 2024-08-17T17:07:02.628 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.6109fc6c8d91 2024-08-17T17:07:02.628 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.6109fc6c8d91\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:02.628 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse1 2024-08-17T17:07:02.628 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse1 2024-08-17T17:07:02.629 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:02.629 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse1 2024-08-17T17:07:02.629 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:02.629 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:02.673 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:02.673 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:02.673 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:02.673 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:02.673 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:02.712 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2024-08-17T17:07:02.712 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2024-08-17T17:07:02.712 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-17T17:07:02.712 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:02.755 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-08-17T17:07:02.759 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:02.759 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:02.761 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse1 --format %b 2024-08-17T17:07:02.762 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2024-08-17T17:07:02.763 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse1.out --format %b 2024-08-17T17:07:02.764 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2024-08-17T17:07:02.764 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:02.764 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:02.764 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-17T17:07:02.765 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:02.765 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2024-08-17T17:07:02.891 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-08-17T17:07:02.894 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2024-08-17T17:07:02.937 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2024-08-17T17:07:02.937 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2024-08-17T17:07:02.940 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:07:02.940 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80524/sparse2 2024-08-17T17:07:03.004 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-08-17T17:07:03.005 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:03.005 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:07:03.008 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2024-08-17T17:07:03.008 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse2 2024-08-17T17:07:03.008 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-08-17T17:07:03.052 INFO:tasks.workunit.client.0.smithi064.stdout:sparse2 2 MiB 2 2024-08-17T17:07:03.053 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse2 2024-08-17T17:07:03.053 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse2 2024-08-17T17:07:03.053 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse2 2024-08-17T17:07:03.053 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:03.053 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:03.100 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.6230bc520ed 2024-08-17T17:07:03.100 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.6230bc520ed 2024-08-17T17:07:03.100 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse2 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse2 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.6230bc520ed\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse2 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:03.101 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:03.145 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:03.145 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:03.146 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:03.146 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:03.146 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:02 smithi175 ceph-mon[34606]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:07:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:02 smithi175 ceph-mon[34606]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 187 op/s 2024-08-17T17:07:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:02 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:02 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.186 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=0 2024-08-17T17:07:03.186 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2024-08-17T17:07:03.186 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-08-17T17:07:03.187 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[30574]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:07:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[30574]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 187 op/s 2024-08-17T17:07:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[40964]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:07:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[40964]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 140 KiB/s rd, 14 KiB/s wr, 187 op/s 2024-08-17T17:07:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:02 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/496050711' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-17T17:07:03.231 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-08-17T17:07:03.235 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:03.235 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:03.238 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse2 --format %b 2024-08-17T17:07:03.239 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2024-08-17T17:07:03.239 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse2.out --format %b 2024-08-17T17:07:03.240 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2024-08-17T17:07:03.240 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:03.240 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:03.240 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-17T17:07:03.240 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:03.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2024-08-17T17:07:03.331 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-08-17T17:07:03.334 INFO:tasks.workunit.client.0.smithi064.stderr:+ truncate /tmp/rbd_import_export_80524/sparse1 -s 10M 2024-08-17T17:07:03.335 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 - sparse1 2024-08-17T17:07:03.438 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:07:03.438 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:03.439 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:07:03.442 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2024-08-17T17:07:03.442 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse1 2024-08-17T17:07:03.442 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-08-17T17:07:03.488 INFO:tasks.workunit.client.0.smithi064.stdout:sparse1 10 MiB 2 2024-08-17T17:07:03.489 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse1 2024-08-17T17:07:03.489 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse1 2024-08-17T17:07:03.489 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse1 2024-08-17T17:07:03.489 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:03.490 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:03.535 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.625d42b0e98c 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.625d42b0e98c 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse1 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse1 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.625d42b0e98c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:03.536 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse1 2024-08-17T17:07:03.537 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:03.537 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:03.579 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:03.580 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:03.580 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:03.580 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:03.580 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:03.620 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2024-08-17T17:07:03.620 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2024-08-17T17:07:03.620 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-17T17:07:03.620 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:03.663 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-08-17T17:07:03.667 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:03.667 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/sparse1 /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:03.674 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse1 --format %b 2024-08-17T17:07:03.675 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2024-08-17T17:07:03.675 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse1.out --format %b 2024-08-17T17:07:03.676 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2024-08-17T17:07:03.677 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:03.677 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:03.677 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-17T17:07:03.677 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse1.out 2024-08-17T17:07:03.678 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2024-08-17T17:07:03.764 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-08-17T17:07:03.767 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_80524/sparse2 oflag=append conv=notrunc 2024-08-17T17:07:03.774 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2024-08-17T17:07:03.774 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2024-08-17T17:07:03.775 INFO:tasks.workunit.client.0.smithi064.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00614367 s, 341 MB/s 2024-08-17T17:07:03.775 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 - sparse2 2024-08-17T17:07:03.867 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:07:03.867 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:03.867 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:07:03.870 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2024-08-17T17:07:03.870 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse2 2024-08-17T17:07:03.871 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-08-17T17:07:03.918 INFO:tasks.workunit.client.0.smithi064.stdout:sparse2 4 MiB 2 2024-08-17T17:07:03.919 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse2 2024-08-17T17:07:03.919 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse2 2024-08-17T17:07:03.919 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse2 2024-08-17T17:07:03.919 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:03.919 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:03.967 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.628a60393535 2024-08-17T17:07:03.968 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.628a60393535 2024-08-17T17:07:03.968 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse2 2024-08-17T17:07:03.968 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse2 2024-08-17T17:07:03.968 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.628a60393535\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:03.968 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse2 2024-08-17T17:07:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:04.013 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:04.013 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:04.013 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:04.013 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:04.013 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects='0 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:2 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:3' 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2 3 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-08-17T17:07:04.052 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:04.096 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-08-17T17:07:04.100 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80524/sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:04.100 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_80524/sparse2 /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:04.104 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse2 --format %b 2024-08-17T17:07:04.104 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=6144 2024-08-17T17:07:04.105 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse2.out --format %b 2024-08-17T17:07:04.106 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=6144 2024-08-17T17:07:04.106 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:04.106 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2024-08-17T17:07:04.106 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-17T17:07:04.106 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse2.out 2024-08-17T17:07:04.107 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2024-08-17T17:07:04.196 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-08-17T17:07:04.200 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-08-17T17:07:04.200 INFO:tasks.workunit.client.0.smithi064.stdout:partially-sparse file imports to partially-sparse image 2024-08-17T17:07:04.200 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80524/sparse1 sparse 2024-08-17T17:07:04.268 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-08-17T17:07:04.269 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:04.269 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2024-08-17T17:07:04.272 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2024-08-17T17:07:04.272 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2024-08-17T17:07:04.272 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2024-08-17T17:07:04.272 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:04.272 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:04.315 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.62b7d2666e4f 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.62b7d2666e4f 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.62b7d2666e4f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2024-08-17T17:07:04.316 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:04.317 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:04.359 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:04.360 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:04.360 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:04.360 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:04.360 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:04.399 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2024-08-17T17:07:04.399 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2024-08-17T17:07:04.399 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-17T17:07:04.399 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2024-08-17T17:07:04.489 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-08-17T17:07:04.493 INFO:tasks.workunit.client.0.smithi064.stdout:zeros import through stdin to sparse image 2024-08-17T17:07:04.494 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'zeros import through stdin to sparse image' 2024-08-17T17:07:04.494 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/zero bs=1M count=4 2024-08-17T17:07:04.494 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format - sparse 2024-08-17T17:07:04.546 INFO:tasks.workunit.client.0.smithi064.stderr:4+0 records in 2024-08-17T17:07:04.546 INFO:tasks.workunit.client.0.smithi064.stderr:4+0 records out 2024-08-17T17:07:04.546 INFO:tasks.workunit.client.0.smithi064.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0513544 s, 81.7 MB/s 2024-08-17T17:07:04.555 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2024-08-17T17:07:04.555 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:04.559 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2024-08-17T17:07:04.559 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2024-08-17T17:07:04.559 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2024-08-17T17:07:04.560 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:04.560 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:04.600 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.62d86d5a0b56 2024-08-17T17:07:04.601 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.62d86d5a0b56 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.62d86d5a0b56\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:04.602 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:04.646 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:04.647 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:04.647 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:04.647 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:04.647 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:04.684 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects= 2024-08-17T17:07:04.684 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 2024-08-17T17:07:04.684 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-08-17T17:07:04.684 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2024-08-17T17:07:04.770 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:07:04.773 INFO:tasks.workunit.client.0.smithi064.stdout:zeros export to sparse file 2024-08-17T17:07:04.774 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'zeros export to sparse file' 2024-08-17T17:07:04.774 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd create --new-format sparse --size 4 2024-08-17T17:07:04.818 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-17T17:07:04.821 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info sparse 2024-08-17T17:07:04.821 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep block_name_prefix 2024-08-17T17:07:04.821 INFO:tasks.workunit.client.0.smithi064.stderr:++ awk '{print $NF;}' 2024-08-17T17:07:04.864 INFO:tasks.workunit.client.0.smithi064.stderr:+ prefix=rbd_data.62f92c59c3c7 2024-08-17T17:07:04.864 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/zero bs=4M count=1 2024-08-17T17:07:04.865 INFO:tasks.workunit.client.0.smithi064.stderr:++ get_image_data_pool sparse 2024-08-17T17:07:04.865 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2024-08-17T17:07:04.865 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2024-08-17T17:07:04.865 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep 'data_pool: ' 2024-08-17T17:07:04.865 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk -F: '{ print $NF }' 2024-08-17T17:07:04.908 INFO:tasks.workunit.client.0.smithi064.stderr:++ data_pool= 2024-08-17T17:07:04.908 INFO:tasks.workunit.client.0.smithi064.stderr:++ '[' -z ']' 2024-08-17T17:07:04.908 INFO:tasks.workunit.client.0.smithi064.stderr:++ data_pool=rbd 2024-08-17T17:07:04.908 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo rbd 2024-08-17T17:07:04.908 INFO:tasks.workunit.client.0.smithi064.stderr:+ rados -p rbd put rbd_data.62f92c59c3c7.000000000000 - 2024-08-17T17:07:04.948 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2024-08-17T17:07:04.949 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2024-08-17T17:07:04.949 INFO:tasks.workunit.client.0.smithi064.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0835179 s, 50.2 MB/s 2024-08-17T17:07:04.970 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2024-08-17T17:07:04.970 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2024-08-17T17:07:04.970 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2024-08-17T17:07:04.971 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2024-08-17T17:07:04.971 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2024-08-17T17:07:05.014 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.62f92c59c3c7 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.62f92c59c3c7 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.62f92c59c3c7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2024-08-17T17:07:05.015 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2024-08-17T17:07:05.016 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2024-08-17T17:07:05.016 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2024-08-17T17:07:05.059 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2024-08-17T17:07:05.059 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2024-08-17T17:07:05.059 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2024-08-17T17:07:05.059 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2024-08-17T17:07:05.059 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2024-08-17T17:07:05.097 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=0 2024-08-17T17:07:05.098 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2024-08-17T17:07:05.098 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-08-17T17:07:05.098 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse 2024-08-17T17:07:05.098 INFO:tasks.workunit.client.0.smithi064.stderr:rm: cannot remove '/tmp/rbd_import_export_80524/sparse': No such file or directory 2024-08-17T17:07:05.099 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:07:05.099 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse /tmp/rbd_import_export_80524/sparse 2024-08-17T17:07:05.140 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2024-08-17T17:07:05.144 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_80524/sparse --format=%b 2024-08-17T17:07:05.145 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 = 0 ']' 2024-08-17T17:07:05.145 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2024-08-17T17:07:05.235 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2024-08-17T17:07:05.238 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_80524/sparse /tmp/rbd_import_export_80524/sparse1 /tmp/rbd_import_export_80524/sparse2 /tmp/rbd_import_export_80524/sparse3 2024-08-17T17:07:05.241 INFO:tasks.workunit.client.0.smithi064.stderr:rm: cannot remove '/tmp/rbd_import_export_80524/sparse3': No such file or directory 2024-08-17T17:07:05.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2024-08-17T17:07:05.241 INFO:tasks.workunit.client.0.smithi064.stdout:OK 2024-08-17T17:07:05.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo OK 2024-08-17T17:07:05.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm -rf /tmp/rbd_import_export_80524 2024-08-17T17:07:05.243 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-08-17T17:07:05.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T17:07:05.312 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-08-17T17:07:05.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:07:05.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:05 smithi064 ceph-mon[30574]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 11 KiB/s wr, 149 op/s 2024-08-17T17:07:05.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:05 smithi064 ceph-mon[40964]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 11 KiB/s wr, 149 op/s 2024-08-17T17:07:05.575 DEBUG:teuthology.parallel:result is None 2024-08-17T17:07:05.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:07:05.604 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:07:05.604 DEBUG:teuthology.orchestra.run.smithi064:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:07:05.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:05 smithi175 ceph-mon[34606]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 11 KiB/s wr, 149 op/s 2024-08-17T17:07:05.661 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:07:05.661 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:07:05.662 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-08-17T17:07:05.662 DEBUG:teuthology.parallel:result is None 2024-08-17T17:07:07.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:06 smithi175 ceph-mon[34606]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:07:07.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:06 smithi175 ceph-mon[34606]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 564 op/s 2024-08-17T17:07:07.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:06 smithi064 ceph-mon[30574]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:07:07.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:06 smithi064 ceph-mon[30574]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 564 op/s 2024-08-17T17:07:07.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:06 smithi064 ceph-mon[40964]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:07:07.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:06 smithi064 ceph-mon[40964]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 564 op/s 2024-08-17T17:07:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:09 smithi175 ceph-mon[34606]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2024-08-17T17:07:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:09 smithi064 ceph-mon[30574]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2024-08-17T17:07:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:09 smithi064 ceph-mon[40964]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2024-08-17T17:07:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:11 smithi175 ceph-mon[34606]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:11 smithi175 ceph-mon[34606]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:07:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:11 smithi064 ceph-mon[30574]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:11 smithi064 ceph-mon[30574]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:07:11.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:11 smithi064 ceph-mon[40964]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:11 smithi064 ceph-mon[40964]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:07:12.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:11] "GET /metrics HTTP/1.1" 200 33952 "" "Prometheus/2.43.0" 2024-08-17T17:07:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:13 smithi175 ceph-mon[34606]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2024-08-17T17:07:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:13 smithi064 ceph-mon[30574]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2024-08-17T17:07:13.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:13 smithi064 ceph-mon[40964]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2024-08-17T17:07:15.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:15 smithi175 ceph-mon[34606]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:15 smithi064 ceph-mon[30574]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:15 smithi064 ceph-mon[40964]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 956 KiB/s rd, 1.3 MiB/s wr, 306 op/s 2024-08-17T17:07:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:17 smithi175 ceph-mon[34606]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:17 smithi064 ceph-mon[30574]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:17.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:17 smithi064 ceph-mon[40964]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:17.805 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:18.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:07:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:18 smithi175 ceph-mon[34606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:18 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:18 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:18 smithi175 ceph-mon[34606]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:19.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[30574]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[30574]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[40964]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:19.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:19.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:18 smithi064 ceph-mon[40964]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:19.262 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:07:20.932 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (118s) 88s ago 2m 15.7M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (2m) 0s ago 2m 88.2M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (98s) 88s ago 98s 14.2M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (96s) 0s ago 95s 15.7M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,8765 running (6m) 0s ago 6m 439M - 18.2.4-719-gced649f8 c870753d67c2 19b32cea2da4 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283,8765 running (8m) 88s ago 8m 483M - 18.2.4-719-gced649f8 c870753d67c2 9c4b8c60d597 2024-08-17T17:07:21.513 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (8m) 88s ago 8m 43.8M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (7m) 0s ago 7m 40.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (7m) 88s ago 7m 37.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (2m) 88s ago 2m 15.7M - 1.5.0 0da6a335fe13 495c9a60a04b 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (2m) 0s ago 2m 16.4M - 1.5.0 0da6a335fe13 60b7297393b5 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (6m) 88s ago 6m 50.2M 4016M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (6m) 88s ago 6m 44.2M 4016M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (5m) 88s ago 5m 41.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (5m) 88s ago 5m 48.4M 4016M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (4m) 0s ago 4m 47.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (4m) 0s ago 4m 52.2M 4016M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:07:21.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (4m) 0s ago 4m 56.8M 4016M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:07:21.515 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (3m) 0s ago 3m 52.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:07:21.515 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (113s) 0s ago 3m 43.1M - 2.43.0 a07b618ecd1d a37a62227643 2024-08-17T17:07:21.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[30574]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:21.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[40964]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:21.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:21 smithi175 ceph-mon[34606]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:21 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:21 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:07:21.849 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:07:21.850 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 15 2024-08-17T17:07:21.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:07:21.851 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:07:21.851 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:21] "GET /metrics HTTP/1.1" 200 33952 "" "Prometheus/2.43.0" 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "", 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "", 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:07:22.134 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:07:22.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[30574]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[30574]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/441252257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:22.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[40964]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[40964]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:22 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/441252257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:22 smithi175 ceph-mon[34606]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:22 smithi175 ceph-mon[34606]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:22 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/441252257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:23 smithi175 ceph-mon[34606]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:23 smithi175 ceph-mon[34606]: from='client.25421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:23 smithi175 ceph-mon[34606]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[30574]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[30574]: from='client.25421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[30574]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[40964]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[40964]: from='client.25421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:23 smithi064 ceph-mon[40964]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:07:25.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:25.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:26 smithi175 ceph-mon[34606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:26 smithi064 ceph-mon[30574]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:26.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:26 smithi064 ceph-mon[40964]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:07:27.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:27 smithi064 ceph-mon[30574]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:27.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:27 smithi064 ceph-mon[40964]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:27.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:27 smithi175 ceph-mon[34606]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:29.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:29 smithi175 ceph-mon[34606]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:29.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:29 smithi064 ceph-mon[30574]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:29.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:29 smithi064 ceph-mon[40964]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:31 smithi175 ceph-mon[34606]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:31.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:31 smithi064 ceph-mon[30574]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:31 smithi064 ceph-mon[40964]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:32.102 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:31 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:31] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.43.0" 2024-08-17T17:07:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:32 smithi064 ceph-mon[30574]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:32 smithi064 ceph-mon[40964]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:32 smithi175 ceph-mon[34606]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:35.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:35 smithi175 ceph-mon[34606]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:07:35.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[30574]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:07:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:07:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:35 smithi064 ceph-mon[40964]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:07:36.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:36 smithi064 ceph-mon[30574]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi175 2024-08-17T17:07:36.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:36 smithi064 ceph-mon[40964]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi175 2024-08-17T17:07:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:36 smithi175 ceph-mon[34606]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi175 2024-08-17T17:07:37.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:37 smithi064 ceph-mon[30574]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:37.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:37 smithi064 ceph-mon[40964]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:37 smithi175 ceph-mon[34606]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:39 smithi175 ceph-mon[34606]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:39.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:39 smithi064 ceph-mon[30574]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:39 smithi064 ceph-mon[40964]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:40 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:40 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:40 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:41 smithi175 ceph-mon[34606]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:41 smithi064 ceph-mon[30574]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:41 smithi064 ceph-mon[40964]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:42.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:41] "GET /metrics HTTP/1.1" 200 33956 "" "Prometheus/2.43.0" 2024-08-17T17:07:43.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:43 smithi175 ceph-mon[34606]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:43 smithi064 ceph-mon[30574]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:43.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:43 smithi064 ceph-mon[40964]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:45.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:45 smithi175 ceph-mon[34606]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:45.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:45 smithi064 ceph-mon[30574]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:45.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:45 smithi064 ceph-mon[40964]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:47.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:47 smithi064 ceph-mon[30574]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:47 smithi064 ceph-mon[40964]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:47.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:47 smithi175 ceph-mon[34606]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:49 smithi175 ceph-mon[34606]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:49 smithi064 ceph-mon[30574]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:49 smithi064 ceph-mon[40964]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:51.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:51 smithi175 ceph-mon[34606]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:51 smithi064 ceph-mon[30574]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:51 smithi064 ceph-mon[40964]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:52.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:07:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:07:51] "GET /metrics HTTP/1.1" 200 33956 "" "Prometheus/2.43.0" 2024-08-17T17:07:52.444 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:07:52.998 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:07:52.998 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (2m) 28s ago 3m 27.9M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:07:52.998 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (3m) 31s ago 3m 88.2M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:07:52.998 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (2m) 28s ago 2m 14.9M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:07:52.998 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (2m) 31s ago 2m 15.7M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,8765 running (7m) 31s ago 7m 439M - 18.2.4-719-gced649f8 c870753d67c2 19b32cea2da4 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283,8765 running (8m) 28s ago 8m 489M - 18.2.4-719-gced649f8 c870753d67c2 9c4b8c60d597 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (8m) 28s ago 9m 46.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (7m) 31s ago 7m 40.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (7m) 28s ago 7m 38.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (3m) 28s ago 3m 15.7M - 1.5.0 0da6a335fe13 495c9a60a04b 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (3m) 31s ago 3m 16.4M - 1.5.0 0da6a335fe13 60b7297393b5 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (7m) 28s ago 6m 55.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (6m) 28s ago 6m 51.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (6m) 28s ago 6m 45.6M 4016M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (5m) 28s ago 5m 50.1M 4016M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (5m) 31s ago 5m 47.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:07:52.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (5m) 31s ago 5m 52.2M 4016M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:07:53.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (4m) 31s ago 4m 56.8M 4016M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:07:53.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (4m) 31s ago 4m 52.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:07:53.000 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (2m) 31s ago 3m 43.1M - 2.43.0 a07b618ecd1d a37a62227643 2024-08-17T17:07:53.332 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:07:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:07:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 15 2024-08-17T17:07:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:07:53.334 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "0/2 daemons upgraded", 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d image on host smithi175", 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:07:53.615 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:07:53.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:53 smithi064 ceph-mon[30574]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:53.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:53 smithi064 ceph-mon[40964]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:53 smithi175 ceph-mon[34606]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:54 smithi175 ceph-mon[34606]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:54 smithi175 ceph-mon[34606]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:54 smithi175 ceph-mon[34606]: from='client.25439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:54 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/2272910319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[30574]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[30574]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[30574]: from='client.25439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/2272910319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[40964]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[40964]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[40964]: from='client.25439 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:54 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/2272910319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:07:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:55 smithi175 ceph-mon[34606]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:55 smithi175 ceph-mon[34606]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:55 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[30574]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[30574]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[40964]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[40964]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:55 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:07:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:57 smithi175 ceph-mon[34606]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:57 smithi064 ceph-mon[30574]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:57 smithi064 ceph-mon[40964]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:07:59 smithi175 ceph-mon[34606]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:07:59 smithi064 ceph-mon[30574]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:07:59.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:07:59 smithi064 ceph-mon[40964]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:01 smithi175 ceph-mon[34606]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:01 smithi064 ceph-mon[30574]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:01.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:01 smithi064 ceph-mon[40964]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:02.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:08:01] "GET /metrics HTTP/1.1" 200 33950 "" "Prometheus/2.43.0" 2024-08-17T17:08:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:03 smithi175 ceph-mon[34606]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:03 smithi064 ceph-mon[30574]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:03 smithi064 ceph-mon[40964]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:05 smithi175 ceph-mon[34606]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:05.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:05 smithi064 ceph-mon[30574]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:05 smithi064 ceph-mon[40964]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:07 smithi175 ceph-mon[34606]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:07 smithi064 ceph-mon[30574]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:07.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:07 smithi064 ceph-mon[40964]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:09 smithi175 ceph-mon[34606]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:09 smithi064 ceph-mon[30574]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:09 smithi064 ceph-mon[40964]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:10 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:10 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:10 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:11.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:11 smithi175 ceph-mon[34606]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:11 smithi064 ceph-mon[30574]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:11 smithi064 ceph-mon[40964]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:12.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:11 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:08:11] "GET /metrics HTTP/1.1" 200 33950 "" "Prometheus/2.43.0" 2024-08-17T17:08:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:12 smithi175 ceph-mon[34606]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:12 smithi064 ceph-mon[30574]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:12 smithi064 ceph-mon[40964]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: Upgrade: Updating mgr.x 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: Deploying daemon mgr.x on smithi175 2024-08-17T17:08:15.338 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:15 smithi175 ceph-mon[34606]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: Upgrade: Updating mgr.x 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: Deploying daemon mgr.x on smithi175 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[30574]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:15.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: Upgrade: Updating mgr.x 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: Deploying daemon mgr.x on smithi175 2024-08-17T17:08:15.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:15 smithi064 ceph-mon[40964]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:15.890 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:15 smithi175 systemd[1]: Stopping Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:16.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:15 smithi175 podman[67903]: 2024-08-17 17:08:15.8892719 +0000 UTC m=+0.127590475 container died 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-08-17T17:08:16.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 podman[67903]: 2024-08-17 17:08:16.5333683 +0000 UTC m=+0.771686873 container cleanup 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:08:16.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 bash[67903]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:08:17.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 podman[67915]: 2024-08-17 17:08:16.809909708 +0000 UTC m=+0.924694280 container remove 19b32cea2da49119e676ab62b1faa9150c918cefea32903da8f7ed01bc6bdd8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-08-17T17:08:17.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:08:17.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:08:17.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 systemd[1]: Stopped Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:17.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:16 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Consumed 10.848s CPU time. 2024-08-17T17:08:17.510 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:17 smithi175 ceph-mon[34606]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:17.510 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 systemd[1]: Starting Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:17.510 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 podman[68050]: 2024-08-17 17:08:17.302064776 +0000 UTC m=+0.017275267 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:08:17.510 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 podman[68050]: 2024-08-17 17:08:17.402731876 +0000 UTC m=+0.117942397 container create 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:08:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:17 smithi064 ceph-mon[30574]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:17 smithi064 ceph-mon[40964]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:17.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 podman[68050]: 2024-08-17 17:08:17.613587911 +0000 UTC m=+0.328798390 container init 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T17:08:17.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 podman[68050]: 2024-08-17 17:08:17.616643254 +0000 UTC m=+0.331853734 container start 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:08:17.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 ceph-mgr[68070]: -- 172.21.15.175:0/1307228061 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564be7d910e0 con 0x564be7d6e800 2024-08-17T17:08:17.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 bash[68050]: 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 2024-08-17T17:08:17.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:17 smithi175 systemd[1]: Started Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:19.022 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:18 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.022 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:18 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.022 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:18 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:19.022 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:18 smithi175 ceph-mon[34606]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[40964]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:19.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:19.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:18 smithi064 ceph-mon[30574]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:21.395 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:21 smithi175 ceph-mon[34606]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:21 smithi064 ceph-mon[30574]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:21 smithi064 ceph-mon[40964]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:22.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:21 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:08:21] "GET /metrics HTTP/1.1" 200 33950 "" "Prometheus/2.43.0" 2024-08-17T17:08:23.142 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.142 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-mon[34606]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:22] ENGINE Bus STARTING 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: CherryPy Checker: 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: The Application mounted at '' has an empty config. 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:22 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:23] ENGINE Serving on http://:::9283 2024-08-17T17:08:23.143 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:23] ENGINE Bus STARTED 2024-08-17T17:08:23.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[30574]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[40964]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:22 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:23.939 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: Standby manager daemon x restarted 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:08:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:23 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:24.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: Standby manager daemon x restarted 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: Standby manager daemon x started 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: Standby manager daemon x restarted 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:08:24.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:08:24.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:24.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:08:24.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:23 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:08:24.489 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (3m) 59s ago 3m 27.9M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (3m) 2s ago 3m 86.7M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (2m) 59s ago 2m 14.9M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (2m) 2s ago 2m 15.9M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (7s) 2s ago 7m 49.7M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1cda34e43a32 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283,8765 running (9m) 59s ago 9m 489M - 18.2.4-719-gced649f8 c870753d67c2 9c4b8c60d597 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (9m) 59s ago 9m 46.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (8m) 2s ago 8m 38.0M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (8m) 59s ago 8m 38.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (3m) 59s ago 3m 15.7M - 1.5.0 0da6a335fe13 495c9a60a04b 2024-08-17T17:08:24.490 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (3m) 2s ago 3m 16.1M - 1.5.0 0da6a335fe13 60b7297393b5 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (7m) 59s ago 7m 55.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (7m) 59s ago 7m 51.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (6m) 59s ago 6m 45.6M 4016M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (6m) 59s ago 6m 50.1M 4016M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (5m) 2s ago 5m 47.8M 4016M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (5m) 2s ago 5m 52.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (5m) 2s ago 5m 57.1M 4016M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (4m) 2s ago 4m 52.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:08:24.491 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (2m) 2s ago 4m 52.1M - 2.43.0 a07b618ecd1d a37a62227643 2024-08-17T17:08:24.821 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:24.822 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 14, 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:08:24.823 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "1/2 daemons upgraded", 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mgr daemons", 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:08:25.109 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[30574]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[30574]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[30574]: from='client.25466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[30574]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1235873202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[40964]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[40964]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[40964]: from='client.25466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[40964]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:25.111 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:24 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1235873202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:25.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:24 smithi175 ceph-mon[34606]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T17:08:25.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:24 smithi175 ceph-mon[34606]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:24 smithi175 ceph-mon[34606]: from='client.25466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:25.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:24 smithi175 ceph-mon[34606]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:25.128 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:24 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1235873202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:25.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:25] ENGINE Bus STOPPING 2024-08-17T17:08:25.920 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:25.624+0000 7f8e74b7e640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-17T17:08:25.920 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setuser ceph since I am not root 2024-08-17T17:08:25.920 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ignoring --setgroup ceph since I am not root 2024-08-17T17:08:25.921 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:25.774+0000 7f98653e5040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:08:26.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:25.926+0000 7f98653e5040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:08:26.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:25.996+0000 7f98653e5040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: Activating manager daemon x 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: mgrmap e23: x(active, starting, since 0.00728822s) 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:08:26.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:08:26.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: Activating manager daemon x 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: mgrmap e23: x(active, starting, since 0.00728822s) 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:08:26.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:08:26.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:25 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:08:26.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:08:26.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:26] ENGINE Bus STOPPED 2024-08-17T17:08:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: Activating manager daemon x 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:08:26.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.24338 172.21.15.64:0/2455886610' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: mgrmap e23: x(active, starting, since 0.00728822s) 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:08:26.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:08:26.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:25 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:08:26.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:26.248+0000 7f98653e5040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:08:26.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:26.310+0000 7f98653e5040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:08:27.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:26 smithi064 ceph-mon[30574]: Manager daemon x is now available 2024-08-17T17:08:27.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:26 smithi064 ceph-mon[40964]: Manager daemon x is now available 2024-08-17T17:08:27.356 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:26 smithi175 ceph-mon[34606]: Manager daemon x is now available 2024-08-17T17:08:27.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:27] ENGINE Bus STARTING 2024-08-17T17:08:27.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.393+0000 7fade8a00640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.393+0000 7fade8a00640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.393+0000 7fade8a00640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.393+0000 7fade8a00640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.393+0000 7fade8a00640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.394+0000 7fade41f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.394+0000 7fade41f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.394+0000 7fade41f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.394+0000 7fade41f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:27.394+0000 7fade41f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:27] ENGINE Serving on http://:::9283 2024-08-17T17:08:27.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:08:27] ENGINE Bus STARTED 2024-08-17T17:08:27.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.282+0000 7f98653e5040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:08:27.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.407+0000 7f98653e5040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:08:27.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.466+0000 7f98653e5040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:08:27.980 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.685+0000 7f98653e5040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:08:27.980 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.753+0000 7f98653e5040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:08:27.980 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.824+0000 7f98653e5040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: mgrmap e24: x(active, since 1.30771s) 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: Checking for cert/key for grafana.a 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: Migrating grafana.a cert to cert store 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: Migrating grafana.a key to cert store 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[40964]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:27.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: mgrmap e24: x(active, since 1.30771s) 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: Checking for cert/key for grafana.a 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: Migrating grafana.a cert to cert store 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: Migrating grafana.a key to cert store 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:27.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:27.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-mon[30574]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: mgrmap e24: x(active, since 1.30771s) 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: Checking for cert/key for grafana.a 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: Migrating grafana.a cert to cert store 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: Migrating grafana.a key to cert store 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:08:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:27 smithi175 ceph-mon[34606]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:28.417 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:27.978+0000 7f98653e5040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:08:28.690 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:28.415+0000 7f98653e5040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:08:28.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:28.532Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:08:28.912 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:28.533Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:08:28.912 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:28.533Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:08:28.912 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:28.535Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:08:28.912 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:28.535Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:08:28.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:28.565+0000 7fadd0f4e640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:08:28] ENGINE Error in HTTPServer.serve 2024-08-17T17:08:28.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: Traceback (most recent call last): 2024-08-17T17:08:28.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:08:28.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: self._connections.run(self.expiration_interval) 2024-08-17T17:08:28.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: self._run(expiration_interval) 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: s = self.context.wrap_socket( 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: return self.sslsocket_class._create( 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: self.do_handshake() 2024-08-17T17:08:28.913 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:08:28.914 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: self._sslobj.do_handshake() 2024-08-17T17:08:28.914 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:08:28.914 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:08:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[30574]: Deploying cephadm binary to smithi175 2024-08-17T17:08:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[30574]: Deploying cephadm binary to smithi064 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[30574]: [17/Aug/2024:17:08:28] ENGINE Bus STARTING 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[30574]: [17/Aug/2024:17:08:28] ENGINE Serving on https://172.21.15.175:7150 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[30574]: [17/Aug/2024:17:08:28] ENGINE Error in HTTPServer.serve 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:08:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:08:29.189 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:08:29.190 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:28.746+0000 7f98653e5040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:08:29.190 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:28.816+0000 7f98653e5040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:08:29.190 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:28.970+0000 7f98653e5040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:08:29.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[40964]: Deploying cephadm binary to smithi175 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[40964]: Deploying cephadm binary to smithi064 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[40964]: [17/Aug/2024:17:08:28] ENGINE Bus STARTING 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[40964]: [17/Aug/2024:17:08:28] ENGINE Serving on https://172.21.15.175:7150 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:28 smithi064 ceph-mon[40964]: [17/Aug/2024:17:08:28] ENGINE Error in HTTPServer.serve 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:08:29.191 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:08:29.192 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:08:29.193 INFO:journalctl@ceph.mon.c.smithi064.stdout: self.do_handshake() 2024-08-17T17:08:29.193 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:08:29.193 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:08:29.193 INFO:journalctl@ceph.mon.c.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-mon[34606]: Deploying cephadm binary to smithi175 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-mon[34606]: Deploying cephadm binary to smithi064 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-mon[34606]: [17/Aug/2024:17:08:28] ENGINE Bus STARTING 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-mon[34606]: [17/Aug/2024:17:08:28] ENGINE Serving on https://172.21.15.175:7150 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:28 smithi175 ceph-mon[34606]: [17/Aug/2024:17:08:28] ENGINE Error in HTTPServer.serve 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout: Traceback (most recent call last): 2024-08-17T17:08:29.268 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._run(expiration_interval) 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: s = self.context.wrap_socket( 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: return self.sslsocket_class._create( 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:08:29.269 INFO:journalctl@ceph.mon.b.smithi175.stdout: self.do_handshake() 2024-08-17T17:08:29.270 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:08:29.270 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._sslobj.do_handshake() 2024-08-17T17:08:29.270 INFO:journalctl@ceph.mon.b.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:08:29.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.291+0000 7f98653e5040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:08:29.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.355+0000 7f98653e5040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:08:29.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.551+0000 7f98653e5040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:08:29.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.710+0000 7f98653e5040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T17:08:29.888 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.789+0000 7f98653e5040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:08:29.888 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.886+0000 7f98653e5040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:08:30.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:29.951+0000 7f98653e5040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[30574]: [17/Aug/2024:17:08:28] ENGINE Serving on http://172.21.15.175:8765 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[30574]: [17/Aug/2024:17:08:28] ENGINE Bus STARTED 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[30574]: mgrmap e25: x(active, since 3s) 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[30574]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[40964]: [17/Aug/2024:17:08:28] ENGINE Serving on http://172.21.15.175:8765 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[40964]: [17/Aug/2024:17:08:28] ENGINE Bus STARTED 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[40964]: mgrmap e25: x(active, since 3s) 2024-08-17T17:08:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:29 smithi064 ceph-mon[40964]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:29 smithi175 ceph-mon[34606]: [17/Aug/2024:17:08:28] ENGINE Serving on http://172.21.15.175:8765 2024-08-17T17:08:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:29 smithi175 ceph-mon[34606]: [17/Aug/2024:17:08:28] ENGINE Bus STARTED 2024-08-17T17:08:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:29 smithi175 ceph-mon[34606]: mgrmap e25: x(active, since 3s) 2024-08-17T17:08:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:29 smithi175 ceph-mon[34606]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:30.800 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:30.516+0000 7f98653e5040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:08:30.800 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:30.693+0000 7f98653e5040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:08:30] ENGINE Bus STARTING 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: CherryPy Checker: 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: The Application mounted at '' has an empty config. 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:08:30] ENGINE Serving on http://:::9283 2024-08-17T17:08:30.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: [17/Aug/2024:17:08:30] ENGINE Bus STARTED 2024-08-17T17:08:31.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:30 smithi175 ceph-mon[34606]: Standby manager daemon y started 2024-08-17T17:08:31.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:08:31.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:31.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:08:31.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:31.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[30574]: Standby manager daemon y started 2024-08-17T17:08:31.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:08:31.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:31.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:08:31.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:31.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[40964]: Standby manager daemon y started 2024-08-17T17:08:31.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:08:31.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:08:31.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:08:31.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/3457886605' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:08:32.000 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:31 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:08:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:08:32.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:32 smithi175 ceph-mon[34606]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:08:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:32 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:08:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:32 smithi175 ceph-mon[34606]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[30574]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:08:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:08:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[30574]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[40964]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:08:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:08:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:32 smithi064 ceph-mon[40964]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:33 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:33 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:34 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi064 to 2480M 2024-08-17T17:08:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:34 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:34 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:34 smithi175 ceph-mon[34606]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 458 B/s wr, 11 op/s 2024-08-17T17:08:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi064 to 2480M 2024-08-17T17:08:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[30574]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 458 B/s wr, 11 op/s 2024-08-17T17:08:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi064 to 2480M 2024-08-17T17:08:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:34 smithi064 ceph-mon[40964]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 458 B/s wr, 11 op/s 2024-08-17T17:08:35.973 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:35 smithi175 ceph-mon[34606]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 353 B/s wr, 8 op/s 2024-08-17T17:08:36.115 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:35 smithi064 ceph-mon[30574]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 353 B/s wr, 8 op/s 2024-08-17T17:08:36.115 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:35 smithi064 ceph-mon[40964]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 353 B/s wr, 8 op/s 2024-08-17T17:08:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Adjusting osd_memory_target on smithi175 to 2480M 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:38.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 287 B/s wr, 7 op/s 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:37 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Adjusting osd_memory_target on smithi175 to 2480M 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 287 B/s wr, 7 op/s 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Adjusting osd_memory_target on smithi175 to 2480M 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:38.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 287 B/s wr, 7 op/s 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:38.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:37 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:38 smithi175 ceph-mon[34606]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:08:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:38 smithi175 ceph-mon[34606]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:08:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:38 smithi064 ceph-mon[30574]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:08:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:38 smithi064 ceph-mon[30574]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:08:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:38 smithi064 ceph-mon[40964]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:08:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:38 smithi064 ceph-mon[40964]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:08:40.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:39 smithi175 ceph-mon[34606]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:40.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:39 smithi064 ceph-mon[30574]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:40.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:39 smithi064 ceph-mon[40964]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:42.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:41 smithi175 ceph-mon[34606]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:42.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:41 smithi064 ceph-mon[30574]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:42.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:41 smithi064 ceph-mon[40964]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:42.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[30802]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:08:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:08:43.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:42 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:43.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:42 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:43.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:42 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:44.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:43 smithi175 ceph-mon[34606]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:44.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:43 smithi064 ceph-mon[40964]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:43 smithi064 ceph-mon[30574]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-17T17:08:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:45 smithi175 ceph-mon[34606]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:45 smithi064 ceph-mon[30574]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:46.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:45 smithi064 ceph-mon[40964]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:48.074 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:47 smithi175 ceph-mon[34606]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:48.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:47 smithi064 ceph-mon[30574]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:48.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:47 smithi064 ceph-mon[40964]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:49.772 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:49 smithi175 ceph-mon[34606]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:50.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:49 smithi064 ceph-mon[30574]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:50.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:49 smithi064 ceph-mon[40964]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:51.127 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:50 smithi175 systemd[1]: Stopping Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:51.411 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.126Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:08:51.411 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:08:51.411 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:08:51.411 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:08:51.412 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:08:51.413 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[64783]: ts=2024-08-17T17:08:51.127Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:08:51.413 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 podman[75932]: 2024-08-17 17:08:51.13712132 +0000 UTC m=+0.110728133 container died a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:51.879 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:51 smithi175 ceph-mon[34606]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:51.879 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 podman[75932]: 2024-08-17 17:08:51.610610489 +0000 UTC m=+0.584217301 container cleanup a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:51.879 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 bash[75932]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:08:52.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:51 smithi175 podman[75948]: 2024-08-17 17:08:51.878565774 +0000 UTC m=+0.743639609 container remove a37a622276436fd198d880d4d4c14e34681b41d4cece5b4784b06a2a7d6cd777 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:52.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:08:52.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 systemd[1]: Stopped Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:52.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.676s CPU time. 2024-08-17T17:08:52.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 systemd[1]: Starting Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:51 smithi064 ceph-mon[30574]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:51 smithi064 ceph-mon[40964]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:52.508 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 podman[76036]: 2024-08-17 17:08:52.25470822 +0000 UTC m=+0.103471664 container create a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:52.508 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 podman[76036]: 2024-08-17 17:08:52.170073579 +0000 UTC m=+0.018837046 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:08:52.508 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 podman[76036]: 2024-08-17 17:08:52.465455372 +0000 UTC m=+0.314218823 container init a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 podman[76036]: 2024-08-17 17:08:52.468136538 +0000 UTC m=+0.316899987 container start a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.491Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.491Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.491Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi175 (none))" 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.491Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.491Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.494Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.494Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.495Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.495Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:08:52.509 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.497Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.497Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=9.094µs 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.497Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.498Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.507Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.507Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-17T17:08:52.510 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.507Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=128.789µs wal_replay_duration=10.370417ms wbl_replay_duration=109ns total_replay_duration=10.682114ms 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.509Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.509Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.509Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.530Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.906215ms db_storage=860ns remote_storage=1.015µs web_handler=569ns query_engine=648ns scrape=388.992µs scrape_sd=102.218µs notify=10.874µs notify_sd=10.61µs rules=20.06176ms tracing=5.998µs 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.530Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:08:52.530Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 bash[76036]: a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 2024-08-17T17:08:52.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:08:52 smithi175 systemd[1]: Started Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:53 smithi175 ceph-mon[34606]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[40964]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:53 smithi064 ceph-mon[30574]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:54.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:54 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:08:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:54 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:08:55.426 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:08:55.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:55 smithi064 ceph-mon[30574]: Upgrade: Updating mgr.y 2024-08-17T17:08:55.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:55 smithi064 ceph-mon[30574]: Deploying daemon mgr.y on smithi064 2024-08-17T17:08:55.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:55 smithi064 ceph-mon[40964]: Upgrade: Updating mgr.y 2024-08-17T17:08:55.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:55 smithi064 ceph-mon[40964]: Deploying daemon mgr.y on smithi064 2024-08-17T17:08:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:55 smithi175 ceph-mon[34606]: Upgrade: Updating mgr.y 2024-08-17T17:08:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:55 smithi175 ceph-mon[34606]: Deploying daemon mgr.y on smithi064 2024-08-17T17:08:55.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:55 smithi064 systemd[1]: Stopping Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:56.052 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:08:56.052 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (3m) 23s ago 4m 28.0M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:08:56.052 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (4m) 23s ago 4m 86.7M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (3m) 23s ago 3m 15.9M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (3m) 23s ago 3m 16.5M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (38s) 23s ago 8m 494M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1cda34e43a32 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283,8765 running (9m) 23s ago 9m 367M - 18.2.4-719-gced649f8 c870753d67c2 9c4b8c60d597 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (10m) 23s ago 10m 49.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (8m) 23s ago 8m 41.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (8m) 23s ago 8m 41.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (4m) 23s ago 4m 15.4M - 1.5.0 0da6a335fe13 495c9a60a04b 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (4m) 23s ago 4m 16.5M - 1.5.0 0da6a335fe13 60b7297393b5 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (8m) 23s ago 8m 56.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (7m) 23s ago 7m 51.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:08:56.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (7m) 23s ago 7m 46.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (6m) 23s ago 6m 50.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (6m) 23s ago 6m 47.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (6m) 23s ago 6m 52.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (5m) 23s ago 5m 57.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (5m) 23s ago 5m 53.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:08:56.054 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 starting - - - - 2024-08-17T17:08:56.230 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:55 smithi064 podman[86688]: 2024-08-17 17:08:55.954423276 +0000 UTC m=+0.145709777 container died 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:08:56.404 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 14, 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:08:56.405 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:08:56.611 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 podman[86688]: 2024-08-17 17:08:56.321283837 +0000 UTC m=+0.512570334 container cleanup 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:08:56.611 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 bash[86688]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:08:56.708 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:08:56.708 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:08:56.708 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "1/2 daemons upgraded", 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mgr daemons", 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:08:56.709 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:08:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:56 smithi175 ceph-mon[34606]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:56 smithi175 ceph-mon[34606]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:56 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/812137779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:56.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[40964]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:56.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[40964]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:56.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/812137779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[30574]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[30574]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:56 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/812137779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 podman[86719]: 2024-08-17 17:08:56.6420027 +0000 UTC m=+0.683396714 container remove 9c4b8c60d597da47029eaabf05ef875fc3aa97ef6ce5dee83abc647de9009ee5 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 systemd[1]: Stopped Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:56.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:56 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service: Consumed 48.094s CPU time. 2024-08-17T17:08:57.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 systemd[1]: Starting Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:08:57.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 podman[86894]: 2024-08-17 17:08:57.153090993 +0000 UTC m=+0.017739633 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:08:57.547 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 podman[86894]: 2024-08-17 17:08:57.282763943 +0000 UTC m=+0.147412583 container create af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:08:57.547 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 podman[86894]: 2024-08-17 17:08:57.518525103 +0000 UTC m=+0.383173755 container init af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:08:57.548 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 podman[86894]: 2024-08-17 17:08:57.521565156 +0000 UTC m=+0.386213800 container start af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True) 2024-08-17T17:08:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:57 smithi175 ceph-mon[34606]: from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:57 smithi175 ceph-mon[34606]: from='client.25538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:57 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:57.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[30574]: from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[30574]: from='client.25538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mgr[86913]: -- 172.21.15.64:0/1392421442 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b851e1b0e0 con 0x55b851df9400 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 bash[86894]: af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:08:57 smithi064 systemd[1]: Started Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[40964]: from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[40964]: from='client.25538 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:57 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:08:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:58 smithi175 ceph-mon[34606]: from='client.15654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:58 smithi175 ceph-mon[34606]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:58 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:58 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:58 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[30574]: from='client.15654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[30574]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[40964]: from='client.15654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[40964]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:08:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:58 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:08:59.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:08:59 smithi064 ceph-mon[30574]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:08:59.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:08:59 smithi064 ceph-mon[40964]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:00.019 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:08:59 smithi175 ceph-mon[34606]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:02 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:02 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:03] ENGINE Bus STARTING 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: CherryPy Checker: 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: The Application mounted at '' has an empty config. 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:03] ENGINE Serving on http://:::9283 2024-08-17T17:09:03.297 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:03] ENGINE Bus STARTED 2024-08-17T17:09:03.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: Standby manager daemon y restarted 2024-08-17T17:09:03.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: Standby manager daemon y started 2024-08-17T17:09:03.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:09:03.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:03.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: Standby manager daemon y restarted 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: Standby manager daemon y started 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:03 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: Standby manager daemon y restarted 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: Standby manager daemon y started 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:09:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:03 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:04.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:04 smithi064 ceph-mon[30574]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:04.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:04 smithi064 ceph-mon[30574]: mgrmap e27: x(active, since 38s), standbys: y 2024-08-17T17:09:04.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:04 smithi064 ceph-mon[40964]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:04.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:04 smithi064 ceph-mon[40964]: mgrmap e27: x(active, since 38s), standbys: y 2024-08-17T17:09:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:04 smithi175 ceph-mon[34606]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:04 smithi175 ceph-mon[34606]: mgrmap e27: x(active, since 38s), standbys: y 2024-08-17T17:09:05.144 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:05] ENGINE Bus STOPPING 2024-08-17T17:09:05.144 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:09:05.144 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:05] ENGINE Bus STOPPED 2024-08-17T17:09:05.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:04 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:09:04.990+0000 7fae79e8d640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-17T17:09:05.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: ignoring --setuser ceph since I am not root 2024-08-17T17:09:05.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: ignoring --setgroup ceph since I am not root 2024-08-17T17:09:05.411 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mgr[68070]: -- 172.21.15.175:0/3475518129 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e4b9aef0e0 con 0x55e4b9acd000 2024-08-17T17:09:06.013 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.693+0000 7f57b5b5a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.013 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.693+0000 7f57b5b5a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.013 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.693+0000 7f57b5b5a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.013 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.693+0000 7f57b5b5a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.693+0000 7f57b5b5a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.696+0000 7f57b9361640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.696+0000 7f57b9361640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.696+0000 7f57b9361640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.696+0000 7f57b9361640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:05.696+0000 7f57b9361640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.014 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: Activating manager daemon y 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25147 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: mgrmap e28: y(active, starting, since 0.00849482s) 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:09:06.015 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:09:06.016 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: Manager daemon y is now available 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: Activating manager daemon y 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25147 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: mgrmap e28: y(active, starting, since 0.00849482s) 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:09:06.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: Manager daemon y is now available 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.019 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:05 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 172.21.15.175:0/2988061076' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: Activating manager daemon y 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25147 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:09:06.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: mgrmap e28: y(active, starting, since 0.00849482s) 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:09:06.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:09:06.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: Manager daemon y is now available 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:05 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:09:06.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:06] ENGINE Bus STARTING 2024-08-17T17:09:06.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:06] ENGINE Serving on http://:::9283 2024-08-17T17:09:06.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:09:06] ENGINE Bus STARTED 2024-08-17T17:09:07.365 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:06 smithi175 ceph-mon[34606]: mgrmap e29: y(active, since 1.00893s) 2024-08-17T17:09:07.365 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:06 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:07.365 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:06 smithi175 ceph-mon[34606]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[30574]: mgrmap e29: y(active, since 1.00893s) 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[30574]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[40964]: mgrmap e29: y(active, since 1.00893s) 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:09:07.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:06 smithi064 ceph-mon[40964]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:07.642+0000 7f5796899640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:09:07] ENGINE Error in HTTPServer.serve 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: Traceback (most recent call last): 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._connections.run(self.expiration_interval) 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._run(expiration_interval) 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:09:07.688 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: s = self.context.wrap_socket( 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: return self.sslsocket_class._create( 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self.do_handshake() 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._sslobj.do_handshake() 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:09:07.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:07 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:09:08.275 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:08 smithi175 ceph-mon[34606]: [17/Aug/2024:17:09:07] ENGINE Bus STARTING 2024-08-17T17:09:08.275 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:08 smithi175 ceph-mon[34606]: [17/Aug/2024:17:09:07] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:09:08.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:08 smithi064 ceph-mon[30574]: [17/Aug/2024:17:09:07] ENGINE Bus STARTING 2024-08-17T17:09:08.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:08 smithi064 ceph-mon[30574]: [17/Aug/2024:17:09:07] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:09:08.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:08 smithi064 ceph-mon[40964]: [17/Aug/2024:17:09:07] ENGINE Bus STARTING 2024-08-17T17:09:08.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:08 smithi064 ceph-mon[40964]: [17/Aug/2024:17:09:07] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:09:09.272 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[30574]: [17/Aug/2024:17:09:07] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[30574]: [17/Aug/2024:17:09:07] ENGINE Bus STARTED 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[30574]: [17/Aug/2024:17:09:07] ENGINE Error in HTTPServer.serve 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:09:09.273 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[30574]: mgrmap e30: y(active, since 3s) 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[40964]: [17/Aug/2024:17:09:07] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:09:09.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[40964]: [17/Aug/2024:17:09:07] ENGINE Bus STARTED 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[40964]: [17/Aug/2024:17:09:07] ENGINE Error in HTTPServer.serve 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:09:09.275 INFO:journalctl@ceph.mon.c.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: self.do_handshake() 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:09:09.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:09 smithi064 ceph-mon[40964]: mgrmap e30: y(active, since 3s) 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:09 smithi175 ceph-mon[34606]: [17/Aug/2024:17:09:07] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:09 smithi175 ceph-mon[34606]: [17/Aug/2024:17:09:07] ENGINE Bus STARTED 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:09 smithi175 ceph-mon[34606]: [17/Aug/2024:17:09:07] ENGINE Error in HTTPServer.serve 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: Traceback (most recent call last): 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._run(expiration_interval) 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:09:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: s = self.context.wrap_socket( 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: return self.sslsocket_class._create( 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: self.do_handshake() 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._sslobj.do_handshake() 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:09:09.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:09 smithi175 ceph-mon[34606]: mgrmap e30: y(active, since 3s) 2024-08-17T17:09:10.277 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-mon[34606]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:10.277 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.277 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.278 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:09:10] ENGINE Bus STARTING 2024-08-17T17:09:10.278 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: CherryPy Checker: 2024-08-17T17:09:10.278 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: The Application mounted at '' has an empty config. 2024-08-17T17:09:10.278 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: 2024-08-17T17:09:10.278 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:09:10] ENGINE Serving on http://:::9283 2024-08-17T17:09:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[30574]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[40964]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:10 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:10.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: [17/Aug/2024:17:09:10] ENGINE Bus STARTED 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: mgrmap e31: y(active, since 5s) 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:11 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: mgrmap e31: y(active, since 5s) 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: Standby manager daemon x started 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: mgrmap e31: y(active, since 5s) 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/3415487253' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:11 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[30574]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[30574]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T17:09:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:09:12.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[40964]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:12.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[40964]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T17:09:12.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:12 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:09:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:12 smithi175 ceph-mon[34606]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:12 smithi175 ceph-mon[34606]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T17:09:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:12 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:09:14.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:14 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:14.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:14 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:15 smithi175 ceph-mon[34606]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:09:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[30574]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:09:15.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:15 smithi064 ceph-mon[40964]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:09:16.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:16 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[68066]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:09:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:09:16.564 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:16 smithi175 ceph-mon[34606]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:09:16.565 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:16 smithi175 ceph-mon[34606]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:09:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:16 smithi064 ceph-mon[30574]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:09:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:16 smithi064 ceph-mon[30574]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:09:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:16 smithi064 ceph-mon[40964]: Reconfiguring daemon prometheus.a on smithi175 2024-08-17T17:09:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:16 smithi064 ceph-mon[40964]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:09:18.051 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:17 smithi175 systemd[1]: Stopping Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:18.335 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:09:18.335 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:09:18.335 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.050Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.051Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.051Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[76050]: ts=2024-08-17T17:09:18.051Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:09:18.336 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 podman[78476]: 2024-08-17 17:09:18.060694058 +0000 UTC m=+0.114504337 container died a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:18.615 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:18 smithi175 ceph-mon[34606]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:09:18.615 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 podman[78476]: 2024-08-17 17:09:18.425325931 +0000 UTC m=+0.479136200 container cleanup a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:18.615 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 bash[78476]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:09:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:18 smithi064 ceph-mon[30574]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:09:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:18 smithi064 ceph-mon[40964]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:09:18.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 podman[78490]: 2024-08-17 17:09:18.693725808 +0000 UTC m=+0.634108158 container remove a558da3860565e894b7916e340d7edf1c34143312d01b36abfdd47608c835758 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:18.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:09:18.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 systemd[1]: Stopped Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:18.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.099s CPU time. 2024-08-17T17:09:18.911 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:18 smithi175 systemd[1]: Starting Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:19.324 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 podman[78592]: 2024-08-17 17:09:18.960946105 +0000 UTC m=+0.018387865 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:09:19.324 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 podman[78592]: 2024-08-17 17:09:19.069919344 +0000 UTC m=+0.127361109 container create 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:19.325 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 podman[78592]: 2024-08-17 17:09:19.297174177 +0000 UTC m=+0.354615952 container init 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:19.325 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 podman[78592]: 2024-08-17 17:09:19.299905042 +0000 UTC m=+0.357346810 container start 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:19.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.324Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.324Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.324Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi175 (none))" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.324Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.324Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.326Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.326Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.327Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.328Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.330Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.330Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.638µs 2024-08-17T17:09:19.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.330Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.331Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.340Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.340Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.340Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.340Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.416µs wal_replay_duration=10.736572ms wbl_replay_duration=123ns total_replay_duration=10.775194ms 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.342Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.342Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.342Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.364Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.075928ms db_storage=877ns remote_storage=1.209µs web_handler=373ns query_engine=736ns scrape=363.557µs scrape_sd=91.4µs notify=21.11µs notify_sd=19.797µs rules=21.234398ms tracing=5.72µs 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.364Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:09:19.364Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:09:19.663 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 bash[78592]: 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 2024-08-17T17:09:19.664 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:09:19 smithi175 systemd[1]: Started Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:20 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:09:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:20 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:21 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:21 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:21 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:21 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:09:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:21 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:09:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:21 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:22.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:22 smithi175 ceph-mon[34606]: Upgrade: Updating mgr.x 2024-08-17T17:09:22.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:22 smithi175 ceph-mon[34606]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:22.619 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:22 smithi175 ceph-mon[34606]: Deploying daemon mgr.x on smithi175 2024-08-17T17:09:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[30574]: Upgrade: Updating mgr.x 2024-08-17T17:09:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[30574]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[30574]: Deploying daemon mgr.x on smithi175 2024-08-17T17:09:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[40964]: Upgrade: Updating mgr.x 2024-08-17T17:09:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[40964]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:22 smithi064 ceph-mon[40964]: Deploying daemon mgr.x on smithi175 2024-08-17T17:09:22.905 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:22 smithi175 systemd[1]: Stopping Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:23.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:22 smithi175 podman[78933]: 2024-08-17 17:09:22.904655537 +0000 UTC m=+0.135907974 container died 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:09:23.508 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 podman[78933]: 2024-08-17 17:09:23.264997176 +0000 UTC m=+0.496249614 container cleanup 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T17:09:23.508 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 bash[78933]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 podman[78947]: 2024-08-17 17:09:23.507955181 +0000 UTC m=+0.604840529 container remove 1cda34e43a327c2d19100e9d8051875ef63693783e5d588f923c96c3379ff913 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 systemd[1]: Stopped Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service: Consumed 17.733s CPU time. 2024-08-17T17:09:23.844 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:23 smithi175 systemd[1]: Starting Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:24.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 podman[79076]: 2024-08-17 17:09:23.968697182 +0000 UTC m=+0.017631676 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:09:24.161 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 podman[79076]: 2024-08-17 17:09:24.075817656 +0000 UTC m=+0.124752149 container create 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731) 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:24 smithi175 ceph-mon[34606]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 podman[79076]: 2024-08-17 17:09:24.320455603 +0000 UTC m=+0.369390108 container init 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 podman[79076]: 2024-08-17 17:09:24.323141366 +0000 UTC m=+0.372075861 container start 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 ceph-mgr[79095]: -- 172.21.15.175:0/3897087827 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557bc53cb0e0 con 0x557bc53a9400 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 bash[79076]: 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd 2024-08-17T17:09:24.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:24 smithi175 systemd[1]: Started Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:24 smithi064 ceph-mon[30574]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:24.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:24 smithi064 ceph-mon[40964]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-17T17:09:25.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:25 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:25 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.904 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:25 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:25.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:25.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:25 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:26.797 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:26 smithi175 ceph-mon[34606]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:26 smithi064 ceph-mon[30574]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:26.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:26 smithi064 ceph-mon[40964]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:27.032 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (4m) 18s ago 5m 26.4M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (4m) 17s ago 4m 88.2M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (3m) 18s ago 3m 16.6M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (3m) 17s ago 3m 16.8M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:09:27.618 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 starting - - - - 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (30s) 18s ago 10m 489M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (10m) 18s ago 10m 51.0M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (9m) 17s ago 9m 44.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (9m) 18s ago 9m 44.0M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (4m) 18s ago 4m 15.4M - 1.5.0 0da6a335fe13 495c9a60a04b 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (4m) 17s ago 4m 16.1M - 1.5.0 0da6a335fe13 60b7297393b5 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (8m) 18s ago 8m 56.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (8m) 18s ago 8m 51.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (7m) 18s ago 7m 46.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (7m) 18s ago 7m 51.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (7m) 17s ago 7m 48.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (6m) 17s ago 6m 53.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (6m) 17s ago 6m 56.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:09:27.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (5m) 17s ago 5m 53.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:09:27.620 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 starting - - - - 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:09:27.969 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:09:28.264 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:09:28.264 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:09:28.264 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:09:28.264 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:09:28.265 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:09:28.265 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "", 2024-08-17T17:09:28.265 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mgr daemons", 2024-08-17T17:09:28.265 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:09:28.265 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:09:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:28 smithi175 ceph-mon[34606]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:28 smithi175 ceph-mon[34606]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:28 smithi175 ceph-mon[34606]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:28 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1269770793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[30574]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[30574]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[30574]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1269770793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[40964]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[40964]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[40964]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:28 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1269770793' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.911 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:09:29] ENGINE Bus STARTING 2024-08-17T17:09:29.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: CherryPy Checker: 2024-08-17T17:09:29.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: The Application mounted at '' has an empty config. 2024-08-17T17:09:29.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: 2024-08-17T17:09:29.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:09:29] ENGINE Serving on http://:::9283 2024-08-17T17:09:29.912 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:09:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:09:29] ENGINE Bus STARTED 2024-08-17T17:09:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='client.25610 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:29 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:30.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: Standby manager daemon x restarted 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:30.841 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:30 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: Standby manager daemon x restarted 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: Standby manager daemon x started 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[30574]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: Standby manager daemon x restarted 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:09:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:09:30.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:09:30.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:09:30.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:30 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/2357581282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:09:31.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:31 smithi175 ceph-mon[34606]: mgrmap e33: y(active, since 25s), standbys: x 2024-08-17T17:09:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:31 smithi064 ceph-mon[30574]: mgrmap e33: y(active, since 25s), standbys: x 2024-08-17T17:09:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:31 smithi064 ceph-mon[40964]: mgrmap e33: y(active, since 25s), standbys: x 2024-08-17T17:09:31.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:31 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:09:31] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2024-08-17T17:09:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:32 smithi175 ceph-mon[34606]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:32 smithi064 ceph-mon[30574]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:32.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:32 smithi064 ceph-mon[40964]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all mgr 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:09:33.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all crash 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all rgw 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:09:33.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all nfs 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:09:33.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:33 smithi175 ceph-mon[34606]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all mgr 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all crash 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all rgw 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:09:33.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all mgr 2024-08-17T17:09:33.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all crash 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all rgw 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all nfs 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[40964]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all nfs 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:33.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:09:33.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:33.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:33 smithi064 ceph-mon[30574]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:34.795 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:34 smithi064 systemd[1]: Stopping Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:35.081 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:34 smithi064 ceph-mon[30574]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T17:09:35.081 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:34 smithi064 ceph-mon[30574]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:09:35.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:34 smithi064 ceph-mon[40964]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T17:09:35.082 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:34 smithi064 ceph-mon[40964]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:09:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:34 smithi175 ceph-mon[34606]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T17:09:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:34 smithi175 ceph-mon[34606]: Deploying daemon node-exporter.a on smithi064 2024-08-17T17:09:35.379 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:35 smithi064 podman[95168]: 2024-08-17 17:09:35.080352634 +0000 UTC m=+0.259365769 container died 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:35.632 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:35 smithi064 podman[95168]: 2024-08-17 17:09:35.611752572 +0000 UTC m=+0.790765711 container cleanup 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:35.632 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:35 smithi064 bash[95168]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:09:36.050 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:35 smithi064 ceph-mon[30574]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:36.050 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:36.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:35 smithi064 ceph-mon[40964]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:36.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:36.051 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:35 smithi064 podman[95180]: 2024-08-17 17:09:35.830214329 +0000 UTC m=+0.753418335 container remove 495c9a60a04bfbfc1e3cf18cfa3385f7986a5d1595b4d64a59d91accf98b8d07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:36.051 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:35 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:09:36.051 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:09:36.051 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 systemd[1]: Stopped Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:35 smithi175 ceph-mon[34606]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:36.334 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 2.213s CPU time. 2024-08-17T17:09:36.335 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 systemd[1]: Starting Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:36.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 bash[95307]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:09:37.187 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 bash[95307]: Getting image source signatures 2024-08-17T17:09:37.188 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 bash[95307]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:09:37.188 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 bash[95307]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:09:37.188 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:36 smithi064 bash[95307]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:09:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:38 smithi175 ceph-mon[34606]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:38.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:38 smithi064 ceph-mon[30574]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:38.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:38 smithi064 ceph-mon[40964]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:38.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:38 smithi064 bash[95307]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:09:38.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:38 smithi064 bash[95307]: Writing manifest to image destination 2024-08-17T17:09:39.027 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:38 smithi064 podman[95307]: 2024-08-17 17:09:38.667663401 +0000 UTC m=+2.286900610 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:09:39.027 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:38 smithi064 podman[95307]: 2024-08-17 17:09:38.783091746 +0000 UTC m=+2.402328994 container create 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:39.027 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 podman[95307]: 2024-08-17 17:09:39.026790265 +0000 UTC m=+2.646027491 container init 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:39.438 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 podman[95307]: 2024-08-17 17:09:39.030237025 +0000 UTC m=+2.649474232 container start 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:09:39.438 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.033Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:09:39.438 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.033Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:09:39.438 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.034Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.034Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:09:39.439 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:09:39.440 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:09:39.441 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.035Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:09:39.442 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a[95367]: ts=2024-08-17T17:09:39.036Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:09:39.443 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 bash[95307]: 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 2024-08-17T17:09:39.444 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:09:39 smithi064 systemd[1]: Started Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:40 smithi175 ceph-mon[34606]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:40 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:40 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[30574]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[40964]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:40 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:41.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 systemd[1]: Stopping Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:41 smithi064 ceph-mon[30574]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T17:09:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:41 smithi064 ceph-mon[30574]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:09:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:41 smithi064 ceph-mon[40964]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T17:09:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:41 smithi064 ceph-mon[40964]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:09:41.535 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 podman[82907]: 2024-08-17 17:09:41.278055957 +0000 UTC m=+0.129415818 container died 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:41.536 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:41 smithi175 ceph-mon[34606]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T17:09:41.536 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:41 smithi175 ceph-mon[34606]: Deploying daemon node-exporter.b on smithi175 2024-08-17T17:09:41.902 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 podman[82907]: 2024-08-17 17:09:41.642800745 +0000 UTC m=+0.494160604 container cleanup 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:41.903 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 bash[82907]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:09:42.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 podman[82922]: 2024-08-17 17:09:41.902459971 +0000 UTC m=+0.627825944 container remove 60b7297393b5fae9ea11325dbe9ac17f4ff1be480b84ba29d0f5ebdcdd48d782 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:42.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:41 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:09:42.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:09:42.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 systemd[1]: Stopped Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:42.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 2.123s CPU time. 2024-08-17T17:09:42.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:09:41] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2024-08-17T17:09:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:42 smithi064 ceph-mon[30574]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:42 smithi064 ceph-mon[40964]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:42.442 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:42 smithi175 ceph-mon[34606]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:42.442 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 systemd[1]: Starting Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:09:42.824 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 bash[83048]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:09:42.824 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 bash[83048]: Getting image source signatures 2024-08-17T17:09:43.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 bash[83048]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:09:43.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 bash[83048]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:09:43.161 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:42 smithi175 bash[83048]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:09:44.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:44 smithi064 ceph-mon[30574]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:44 smithi064 ceph-mon[40964]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:44.448 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:44 smithi175 ceph-mon[34606]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:44.911 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:44 smithi175 bash[83048]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:09:44.911 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:44 smithi175 bash[83048]: Writing manifest to image destination 2024-08-17T17:09:45.230 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:44 smithi175 podman[83048]: 2024-08-17 17:09:44.879533617 +0000 UTC m=+2.531462187 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:09:45.231 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:44 smithi175 podman[83048]: 2024-08-17 17:09:44.99503218 +0000 UTC m=+2.646960737 container create ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:45.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 podman[83048]: 2024-08-17 17:09:45.230416605 +0000 UTC m=+2.882345162 container init ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 podman[83048]: 2024-08-17 17:09:45.233447751 +0000 UTC m=+2.885376310 container start ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.237Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:09:45.662 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:09:45.663 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:09:45.664 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:09:45.665 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b[83098]: ts=2024-08-17T17:09:45.238Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 bash[83048]: ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 2024-08-17T17:09:45.666 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:09:45 smithi175 systemd[1]: Started Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:46 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:46 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:48 smithi175 ceph-mon[34606]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:48 smithi064 ceph-mon[40964]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:48 smithi064 ceph-mon[30574]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:50 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:50 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:51.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:51.898 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:51 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.898 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:51 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:51.899 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:51 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:09:51.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:09:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:09:51] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2024-08-17T17:09:52.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:52 smithi064 ceph-mon[30574]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:52.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:52 smithi064 ceph-mon[40964]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:52 smithi175 ceph-mon[34606]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:54 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:54.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:54 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:09:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:56 smithi175 ceph-mon[34606]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:56 smithi175 ceph-mon[34606]: Upgrade: Updating prometheus.a 2024-08-17T17:09:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:56 smithi175 ceph-mon[34606]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:09:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[30574]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[30574]: Upgrade: Updating prometheus.a 2024-08-17T17:09:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[30574]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:09:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[40964]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[40964]: Upgrade: Updating prometheus.a 2024-08-17T17:09:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:56 smithi064 ceph-mon[40964]: Deploying daemon prometheus.a on smithi175 2024-08-17T17:09:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:09:58 smithi175 ceph-mon[34606]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:09:58 smithi064 ceph-mon[30574]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:09:58 smithi064 ceph-mon[40964]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:09:58.587 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (4m) 9s ago 5m 26.4M - 0.25.0 c8568f914cd2 ada422463d74 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (5m) 9s ago 5m 88.4M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (4m) 9s ago 4m 17.1M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (4m) 9s ago 4m 17.1M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (35s) 9s ago 9m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (62s) 9s ago 11m 508M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (11m) 9s ago 11m 52.5M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:09:59.164 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (9m) 9s ago 9m 47.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (9m) 9s ago 9m 46.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (20s) 9s ago 5m 7151k - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (14s) 9s ago 5m 3334k - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (9m) 9s ago 9m 56.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (8m) 9s ago 8m 52.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (8m) 9s ago 8m 46.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (7m) 9s ago 7m 51.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (7m) 9s ago 7m 48.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (7m) 9s ago 7m 53.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (6m) 9s ago 6m 56.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (6m) 9s ago 6m 53.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:09:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (40s) 9s ago 5m 38.1M - 2.43.0 a07b618ecd1d 19a15c572a12 2024-08-17T17:09:59.513 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:09:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:09:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:09:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:09:59.514 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [ 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr" 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T17:09:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading prometheus daemons", 2024-08-17T17:09:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:09:59.811 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:10:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/512408570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[30574]: overall HEALTH_OK 2024-08-17T17:10:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/512408570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:00 smithi064 ceph-mon[40964]: overall HEALTH_OK 2024-08-17T17:10:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/512408570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:00.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:00 smithi175 ceph-mon[34606]: overall HEALTH_OK 2024-08-17T17:10:01.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:01 smithi064 ceph-mon[30574]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:01 smithi064 ceph-mon[40964]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:01.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:01 smithi175 ceph-mon[34606]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:02.169 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:01] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2024-08-17T17:10:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:02 smithi064 ceph-mon[30574]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:02 smithi064 ceph-mon[40964]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:02 smithi175 ceph-mon[34606]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:04 smithi064 ceph-mon[30574]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:04 smithi064 ceph-mon[40964]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:04.474 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:04 smithi175 ceph-mon[34606]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:06.353 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 systemd[1]: Stopping Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:06.353 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-mon[34606]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:06.353 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:06 smithi064 ceph-mon[30574]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:06 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:06 smithi064 ceph-mon[40964]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:06 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:10:06.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.352Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[78607]: ts=2024-08-17T17:10:06.353Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:10:06.662 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 podman[87223]: 2024-08-17 17:10:06.362563813 +0000 UTC m=+0.123652356 container died 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.150 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 podman[87223]: 2024-08-17 17:10:06.693232639 +0000 UTC m=+0.454321180 container cleanup 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.150 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 bash[87223]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:10:07.150 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:06 smithi175 podman[87237]: 2024-08-17 17:10:06.924785423 +0000 UTC m=+0.560086014 container remove 19a15c572a1207bda98ed744dd88d129294a514d7452ca9c793159268ae19470 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.574 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:10:07.574 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 systemd[1]: Stopped Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:07.575 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.125s CPU time. 2024-08-17T17:10:07.575 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 systemd[1]: Starting Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 podman[87365]: 2024-08-17 17:10:07.477313975 +0000 UTC m=+0.018201499 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 podman[87365]: 2024-08-17 17:10:07.586879421 +0000 UTC m=+0.127766919 container create 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 podman[87365]: 2024-08-17 17:10:07.797235642 +0000 UTC m=+0.338123156 container init 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 podman[87365]: 2024-08-17 17:10:07.799898243 +0000 UTC m=+0.340785745 container start 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.819Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.819Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.819Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi175 (none))" 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.819Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:10:07.825 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.819Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:10:07.826 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.821Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:10:07.826 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.822Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T17:10:07.826 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.824Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:10:07.826 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.824Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:10:08.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.825Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:10:08.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.825Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.345µs 2024-08-17T17:10:08.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.825Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:10:08.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.826Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.833Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.833Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.834Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.834Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.834Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.379µs wal_replay_duration=9.215628ms wbl_replay_duration=166ns total_replay_duration=9.286344ms 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.836Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.836Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.836Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.852Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.76138ms db_storage=850ns remote_storage=1.025µs web_handler=649ns query_engine=624ns scrape=451.623µs scrape_sd=112.412µs notify=10.034µs notify_sd=14.043µs rules=14.780994ms tracing=6.11µs 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.852Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:10:07.852Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 bash[87365]: 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 2024-08-17T17:10:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:10:07 smithi175 systemd[1]: Started Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[30574]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[40964]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:08 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:08 smithi175 ceph-mon[34606]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:08 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:08 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:08 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:09.546 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:09 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:09.546 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:09 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:09 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:09 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:09 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:09 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:10.573 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:10 smithi175 ceph-mon[34606]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:10 smithi064 ceph-mon[30574]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:10 smithi064 ceph-mon[40964]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:12 smithi175 ceph-mon[34606]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:12 smithi064 ceph-mon[30574]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:12 smithi064 ceph-mon[40964]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:13.643 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.643 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.643 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.643 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:14.347 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:13] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-08-17T17:10:14.606 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:14 smithi175 ceph-mon[34606]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:14 smithi064 ceph-mon[30574]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:14.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:14 smithi064 ceph-mon[40964]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:16 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:16.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:16 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:17 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:17.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:17 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:17 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:10:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:18 smithi175 ceph-mon[34606]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:18 smithi064 ceph-mon[30574]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:18.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:18 smithi064 ceph-mon[40964]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:19 smithi175 ceph-mon[34606]: Upgrade: Updating alertmanager.a 2024-08-17T17:10:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:19 smithi175 ceph-mon[34606]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:10:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:19 smithi064 ceph-mon[30574]: Upgrade: Updating alertmanager.a 2024-08-17T17:10:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:19 smithi064 ceph-mon[30574]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:10:19.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:19 smithi064 ceph-mon[40964]: Upgrade: Updating alertmanager.a 2024-08-17T17:10:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:19 smithi064 ceph-mon[40964]: Deploying daemon alertmanager.a on smithi064 2024-08-17T17:10:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:20 smithi175 ceph-mon[34606]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:20 smithi064 ceph-mon[30574]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:20 smithi064 ceph-mon[40964]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:21 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:21 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:21.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:21 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:22 smithi175 ceph-mon[34606]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:22 smithi064 ceph-mon[30574]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:22 smithi064 ceph-mon[40964]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:23] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.51.0" 2024-08-17T17:10:24.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:24 smithi064 ceph-mon[30574]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:24.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:24 smithi064 ceph-mon[40964]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:24 smithi175 ceph-mon[34606]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:25.262 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:24 smithi064 systemd[1]: Stopping Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:25.262 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[73856]: ts=2024-08-17T17:10:25.210Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:10:25.677 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:25 smithi064 podman[99614]: 2024-08-17 17:10:25.261015844 +0000 UTC m=+0.136260930 container died ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:25.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:25 smithi064 podman[99614]: 2024-08-17 17:10:25.677110273 +0000 UTC m=+0.552355387 container cleanup ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:25.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:25 smithi064 bash[99614]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:10:26.187 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:25 smithi064 podman[99627]: 2024-08-17 17:10:25.936809863 +0000 UTC m=+0.680120895 container remove ada422463d7494fa923225b55168b42bf6e5a534da0416690121b45963a61bd5 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:26.187 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 podman[99627]: 2024-08-17 17:10:26.012454023 +0000 UTC m=+0.755765054 volume remove b9eed76b9cd5e34d5ede235aad978202f433bd1a8089223efba2a35ceb568ea0 2024-08-17T17:10:26.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:26 smithi175 ceph-mon[34606]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:26.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:26 smithi064 ceph-mon[30574]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:26.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:26 smithi064 ceph-mon[40964]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:26.686 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:10:26.686 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 systemd[1]: Stopped Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:26.686 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 1.193s CPU time. 2024-08-17T17:10:26.686 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 systemd[1]: Starting Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:26.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 podman[99753]: 2024-08-17 17:10:26.587806957 +0000 UTC m=+0.017780279 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-08-17T17:10:26.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 podman[99753]: 2024-08-17 17:10:26.688327538 +0000 UTC m=+0.118300859 volume create 6861d97d78d0d1b117c4047df2afc23745c7d9fbf5050818c8624b5a64c9e4ce 2024-08-17T17:10:26.938 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:26 smithi064 podman[99753]: 2024-08-17 17:10:26.788934038 +0000 UTC m=+0.218907359 container create f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 podman[99753]: 2024-08-17 17:10:27.033383701 +0000 UTC m=+0.463357035 container init f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 podman[99753]: 2024-08-17 17:10:27.036275836 +0000 UTC m=+0.466249150 container start f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.046Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.046Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.047Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.048Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.069Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:10:27.438 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.069Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:10:27.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.071Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:10:27.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:27.071Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:10:27.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 bash[99753]: f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f 2024-08-17T17:10:27.439 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:27 smithi064 systemd[1]: Started Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:28 smithi175 ceph-mon[34606]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:28 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:28 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:28 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[30574]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[40964]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:28 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:29.387 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:29 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:29.048Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000410821s 2024-08-17T17:10:30.150 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:10:30.360 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:30 smithi175 ceph-mon[34606]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:30 smithi064 ceph-mon[30574]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:30 smithi064 ceph-mon[40964]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (4s) 0s ago 6m 17.7M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (5m) 18s ago 5m 88.4M - 9.4.7 954c08fa6188 5977650b0f2d 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (4m) 0s ago 4m 17.4M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (4m) 18s ago 4m 17.2M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (66s) 18s ago 9m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (93s) 0s ago 11m 511M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (11m) 0s ago 11m 51.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:10:30.738 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (10m) 18s ago 10m 43.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (10m) 0s ago 10m 43.4M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (52s) 0s ago 6m 8048k - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (45s) 18s ago 5m 7528k - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (9m) 0s ago 9m 57.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (9m) 0s ago 9m 52.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (8m) 0s ago 8m 47.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (8m) 0s ago 8m 51.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (8m) 18s ago 8m 49.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (7m) 18s ago 7m 53.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (7m) 18s ago 7m 57.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (6m) 18s ago 6m 54.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:10:30.739 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (23s) 18s ago 6m 33.1M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:10:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:10:31.102 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [ 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr" 2024-08-17T17:10:31.420 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:10:31.421 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T17:10:31.421 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading alertmanager daemons", 2024-08-17T17:10:31.421 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:10:31.421 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:10:31.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[30574]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[30574]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/956543883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:31.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[40964]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[40964]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:31 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/956543883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:31 smithi175 ceph-mon[34606]: from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:31 smithi175 ceph-mon[34606]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:31 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:31 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:31 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/956543883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:32 smithi175 ceph-mon[34606]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[40964]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='client.25670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:32 smithi064 ceph-mon[30574]: from='client.15795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:34.291 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:33] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.51.0" 2024-08-17T17:10:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:34 smithi175 ceph-mon[34606]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:34 smithi064 ceph-mon[30574]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:34 smithi064 ceph-mon[40964]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:35.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:35.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:35 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:36.090 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.090 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.090 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:36.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:36.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:36.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:36.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:36.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:35 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:37.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:36 smithi064 ceph-mon[30574]: Upgrade: Updating grafana.a 2024-08-17T17:10:37.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:36 smithi064 ceph-mon[30574]: Deploying daemon grafana.a on smithi175 2024-08-17T17:10:37.051 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:36 smithi064 ceph-mon[40964]: Upgrade: Updating grafana.a 2024-08-17T17:10:37.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:36 smithi064 ceph-mon[40964]: Deploying daemon grafana.a on smithi175 2024-08-17T17:10:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:36 smithi175 ceph-mon[34606]: Upgrade: Updating grafana.a 2024-08-17T17:10:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:36 smithi175 ceph-mon[34606]: Deploying daemon grafana.a on smithi175 2024-08-17T17:10:37.437 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:10:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:10:37.050Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002185357s 2024-08-17T17:10:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:37 smithi175 ceph-mon[34606]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:37 smithi064 ceph-mon[30574]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:38.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:37 smithi064 ceph-mon[40964]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:39.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:38 smithi175 ceph-mon[34606]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T17:10:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:38 smithi064 ceph-mon[40964]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T17:10:39.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:38 smithi064 ceph-mon[30574]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T17:10:40.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:39 smithi175 ceph-mon[34606]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:40.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:39 smithi064 ceph-mon[30574]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:40.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:39 smithi064 ceph-mon[40964]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:42.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:42 smithi175 ceph-mon[34606]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:42 smithi064 ceph-mon[30574]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:42 smithi064 ceph-mon[40964]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:44.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:44 smithi175 ceph-mon[34606]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:44 smithi064 ceph-mon[30574]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:44.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:43] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.51.0" 2024-08-17T17:10:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:44 smithi064 ceph-mon[40964]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:46 smithi175 ceph-mon[34606]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:46 smithi064 ceph-mon[30574]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:46 smithi064 ceph-mon[40964]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:47.007 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:46 smithi175 systemd[1]: Stopping Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:47.306 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 podman[92140]: 2024-08-17 17:10:47.007150043 +0000 UTC m=+0.151983558 container died 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vcs-type=git, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, vendor=Red Hat, Inc., io.buildah.version=1.32.0, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, name=ubi8) 2024-08-17T17:10:47.640 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 podman[92140]: 2024-08-17 17:10:47.413279632 +0000 UTC m=+0.558113144 container cleanup 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, name=ubi8, distribution-scope=public) 2024-08-17T17:10:47.640 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 bash[92140]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:10:47.911 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 podman[92155]: 2024-08-17 17:10:47.639612092 +0000 UTC m=+0.630840251 container remove 5977650b0f2dad5a819f7f4776f4ef9dde4476052859448334bf8e7e30c07562 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, version=8.5, name=ubi8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64) 2024-08-17T17:10:47.911 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:10:47.911 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T17:10:47.911 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 systemd[1]: Stopped Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:47.911 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service: Consumed 7.374s CPU time. 2024-08-17T17:10:48.302 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-mon[34606]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:48.302 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:47 smithi175 systemd[1]: Starting Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:10:48.302 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 podman[92281]: 2024-08-17 17:10:48.081595657 +0000 UTC m=+0.018654986 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-08-17T17:10:48.302 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 podman[92281]: 2024-08-17 17:10:48.207271152 +0000 UTC m=+0.144330475 container create 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:10:48.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:48 smithi064 ceph-mon[40964]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:48.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:48 smithi064 ceph-mon[30574]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:10:48.559 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 podman[92281]: 2024-08-17 17:10:48.409673475 +0000 UTC m=+0.346732813 container init 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:10:48.559 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 podman[92281]: 2024-08-17 17:10:48.412824933 +0000 UTC m=+0.349884255 container start 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:10:48.559 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 bash[92281]: 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 2024-08-17T17:10:48.559 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 systemd[1]: Started Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558613427Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-08-17T17:10:48Z 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558870419Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558920468Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.55892807Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558932623Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558936822Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.5589413Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558945532Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-08-17T17:10:48.835 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558950017Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.5589543Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558958773Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558962912Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558967163Z level=info msg=Target target=[all] 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558975353Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558979291Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558982908Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558986799Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T17:10:48.836 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558990559Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=settings t=2024-08-17T17:10:48.558994481Z level=info msg="App mode production" 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore t=2024-08-17T17:10:48.559361423Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore t=2024-08-17T17:10:48.559381821Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.560170063Z level=info msg="Starting DB migrations" 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.571501Z level=info msg="Executing migration" id="Add uid column to user" 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.577087977Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.584211ms 2024-08-17T17:10:48.837 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.577578977Z level=info msg="Executing migration" id="Update uid column values for users" 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.577753346Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=174.436µs 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.578172899Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.578954388Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=781.147µs 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.579585547Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.585281503Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.693407ms 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.586055197Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.591589494Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.53242ms 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.592208326Z level=info msg="Executing migration" id="Add column uid in team" 2024-08-17T17:10:48.838 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.597428374Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.217678ms 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.59792223Z level=info msg="Executing migration" id="Update uid column values in team" 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.598074992Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=152.919µs 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.600593778Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.60141202Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=818.388µs 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.601856865Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.602550831Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=693.402µs 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.603086368Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-08-17T17:10:48.839 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.608500738Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.41189ms 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.60903707Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.614077843Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.039908ms 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.614498369Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.614539971Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=41.543µs 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.615385525Z level=info msg="Executing migration" id="permission kind migration" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.620425353Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.037867ms 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.620923744Z level=info msg="Executing migration" id="permission attribute migration" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.626103768Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.1784ms 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.6265173Z level=info msg="Executing migration" id="permission identifier migration" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.631957895Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.43843ms 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.632475533Z level=info msg="Executing migration" id="add permission identifier index" 2024-08-17T17:10:48.840 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.633386584Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=911.147µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.633791575Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.634586254Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=794.915µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.634972277Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.635683289Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=713.188µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.636042937Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.636063652Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=20.975µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.636899509Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.637583587Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=683.46µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.637934728Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.638604503Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=668.013µs 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.638947551Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-08-17T17:10:48.841 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.654066253Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.115978ms 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.654577221Z level=info msg="Executing migration" id="create correlation v2" 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.655372231Z level=info msg="Migration successfully executed" id="create correlation v2" duration=794.588µs 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.655733314Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.656463587Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=732.084µs 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.656815101Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.657526543Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=711.909µs 2024-08-17T17:10:48.842 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.657897625Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-08-17T17:10:48.843 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.658614094Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=716.292µs 2024-08-17T17:10:48.843 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.658969878Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-08-17T17:10:48.843 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.659132639Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=162.876µs 2024-08-17T17:10:48.843 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.659800724Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.660380364Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=579.514µs 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.660731353Z level=info msg="Executing migration" id="add provisioning column" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.666083035Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.349392ms 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.666566396Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.666725987Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=159.217µs 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.667363022Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.667407163Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=43.991µs 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.6682268Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.673728903Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.500434ms 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.674174745Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.674387393Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=212.702µs 2024-08-17T17:10:48.844 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.675076043Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.675193183Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=117.046µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.675935677Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.676907045Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=970.739µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.677267628Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.67800929Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=741.429µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.678349008Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.679190175Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=839.858µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.679549795Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.679966749Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=418.137µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.680432477Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.680632022Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=199.648µs 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.681265755Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-08-17T17:10:48.845 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.682067839Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=801.898µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.682419335Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.683128623Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=709.339µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.683465069Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.684154061Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=688.716µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.684497916Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.685218766Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=720.055µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.687397502Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.688217901Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=820.995µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.690138649Z level=info msg="Executing migration" id="create anon_device table" 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.690709335Z level=info msg="Migration successfully executed" id="create anon_device table" duration=570.722µs 2024-08-17T17:10:48.846 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.692847793Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.693550166Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=700.382µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.69570986Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.69659956Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=890.677µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.698564318Z level=info msg="Executing migration" id="create signing_key table" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.699491277Z level=info msg="Migration successfully executed" id="create signing_key table" duration=926.621µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.701289369Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.702092437Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=803.864µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.703963578Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.704647011Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=683.841µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.707514757Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.707958662Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=444.828µs 2024-08-17T17:10:48.847 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.710354066Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.71556041Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.203605ms 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.715934786Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.717589Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.65493ms 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.718780043Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.719712491Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=932.248µs 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.720041115Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.720775202Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=733.895µs 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.721464268Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.722177446Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=713.16µs 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.724381841Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.725254557Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=872.414µs 2024-08-17T17:10:48.848 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.727170412Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.727961634Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=790.996µs 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.729850781Z level=info msg="Executing migration" id="create sso_setting table" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.73062803Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=773.752µs 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.73111937Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.73179959Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=680.3µs 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.732152272Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.732560021Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=407.745µs 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.732896191Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.73293658Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=40.801µs 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.733333223Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.738738765Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.402739ms 2024-08-17T17:10:48.849 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.739234433Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.744604545Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.368514ms 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.745068503Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.745309635Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=241.673µs 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=migrator t=2024-08-17T17:10:48.745682754Z level=info msg="migrations completed" performed=65 skipped=482 duration=174.255839ms 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore t=2024-08-17T17:10:48.746393424Z level=info msg="Created default organization" 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=secrets t=2024-08-17T17:10:48.748548301Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugin.store t=2024-08-17T17:10:48.763451913Z level=info msg="Loading plugins..." 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=local.finder t=2024-08-17T17:10:48.833967733Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugin.store t=2024-08-17T17:10:48.833998713Z level=info msg="Plugins loaded" count=55 duration=70.548024ms 2024-08-17T17:10:48.850 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=query_data t=2024-08-17T17:10:48.837575835Z level=info msg="Query Service initialization" 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=live.push_http t=2024-08-17T17:10:48.841405852Z level=info msg="Live Push Gateway initialization" 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.migration t=2024-08-17T17:10:48.843208623Z level=info msg=Starting 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.state.manager t=2024-08-17T17:10:48.850500658Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=infra.usagestats.collector t=2024-08-17T17:10:48.85190065Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.datasources t=2024-08-17T17:10:48.8547394Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.datasources t=2024-08-17T17:10:48.855054481Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T17:10:49.118 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.alerting t=2024-08-17T17:10:48.886138396Z level=info msg="starting to provision alerting" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.alerting t=2024-08-17T17:10:48.88616063Z level=info msg="finished to provision alerting" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.state.manager t=2024-08-17T17:10:48.886290939Z level=info msg="Warming state cache for startup" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafanaStorageLogger t=2024-08-17T17:10:48.886316489Z level=info msg="Storage starting" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.multiorg.alertmanager t=2024-08-17T17:10:48.886553784Z level=info msg="Starting MultiOrg Alertmanager" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.state.manager t=2024-08-17T17:10:48.887399192Z level=info msg="State cache has been initialized" states=0 duration=1.105883ms 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ngalert.scheduler t=2024-08-17T17:10:48.887632827Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ticker t=2024-08-17T17:10:48.887729552Z level=info msg=starting first_tick=2024-08-17T17:10:50Z 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=http.server t=2024-08-17T17:10:48.88956026Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=http.server t=2024-08-17T17:10:48.889931924Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.dashboard t=2024-08-17T17:10:48.910529918Z level=info msg="starting to provision dashboards" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore.transactions t=2024-08-17T17:10:48.922328016Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana.update.checker t=2024-08-17T17:10:48.964736146Z level=info msg="Update check succeeded" duration=78.408473ms 2024-08-17T17:10:49.119 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugins.update.checker t=2024-08-17T17:10:48.990668271Z level=info msg="Update check succeeded" duration=104.32893ms 2024-08-17T17:10:49.120 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore.transactions t=2024-08-17T17:10:49.015235641Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:10:49.120 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=provisioning.dashboard t=2024-08-17T17:10:49.117192702Z level=info msg="finished to provision dashboards" 2024-08-17T17:10:49.152 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana-apiserver t=2024-08-17T17:10:49.151519682Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-08-17T17:10:49.153 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana-apiserver t=2024-08-17T17:10:49.151899283Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-08-17T17:10:49.892 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.892 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.892 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:49 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:49.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:49 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:50 smithi175 ceph-mon[34606]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:50 smithi064 ceph-mon[30574]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:50 smithi064 ceph-mon[40964]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:51.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:51 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:51 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:51 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:10:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:52 smithi175 ceph-mon[34606]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:52 smithi064 ceph-mon[30574]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:52 smithi064 ceph-mon[40964]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:53 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:53 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:53 smithi175 ceph-mon[34606]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:53 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:53 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:10:53] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.51.0" 2024-08-17T17:10:54.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[30574]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[40964]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:53 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:56 smithi175 ceph-mon[34606]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:56 smithi064 ceph-mon[30574]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:56 smithi064 ceph-mon[40964]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.916 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.917 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:57 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:10:57.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:10:57.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:10:57.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:10:57.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:57 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:10:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:58 smithi175 ceph-mon[34606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:58 smithi175 ceph-mon[34606]: Upgrade: Finalizing container_image settings 2024-08-17T17:10:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:58 smithi175 ceph-mon[34606]: Upgrade: Complete! 2024-08-17T17:10:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:10:58 smithi175 ceph-mon[34606]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[40964]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[40964]: Upgrade: Finalizing container_image settings 2024-08-17T17:10:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[40964]: Upgrade: Complete! 2024-08-17T17:10:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[40964]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:10:58.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[30574]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:10:58.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[30574]: Upgrade: Finalizing container_image settings 2024-08-17T17:10:58.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[30574]: Upgrade: Complete! 2024-08-17T17:10:58.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:10:58 smithi064 ceph-mon[30574]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:00 smithi175 ceph-mon[34606]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:00.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:00 smithi064 ceph-mon[40964]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:00 smithi064 ceph-mon[30574]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:01 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:01 smithi064 ceph-mon[30574]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:01 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:01 smithi064 ceph-mon[40964]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:01 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:01 smithi175 ceph-mon[34606]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:02.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for minority of mons to upgrade' 2024-08-17T17:11:02.649 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:02 smithi064 ceph-mon[30574]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:02.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:02 smithi064 ceph-mon[40964]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:02 smithi175 ceph-mon[34606]: from='client.25688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:03.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:03 smithi064 ceph-mon[30574]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:03.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:03 smithi064 ceph-mon[40964]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:04.154 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:11:03] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.51.0" 2024-08-17T17:11:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:03 smithi175 ceph-mon[34606]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:04.364 INFO:teuthology.orchestra.run.smithi064.stdout:wait for minority of mons to upgrade 2024-08-17T17:11:04.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-17T17:11:05.137 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:06 smithi175 ceph-mon[34606]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:06 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:11:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:06 smithi064 ceph-mon[30574]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:06 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:11:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:06 smithi064 ceph-mon[40964]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:06 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:11:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:08 smithi175 ceph-mon[34606]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:08 smithi175 ceph-mon[34606]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:08 smithi064 ceph-mon[30574]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:08 smithi064 ceph-mon[30574]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:08.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:08 smithi064 ceph-mon[40964]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:08 smithi064 ceph-mon[40964]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:08.984 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:09.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:11:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:10.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:10.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:10.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[30574]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:10.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:09 smithi064 ceph-mon[40964]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:09 smithi175 ceph-mon[34606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:11:11.855 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:11:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:12 smithi175 ceph-mon[34606]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:12 smithi064 ceph-mon[30574]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:12.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:12 smithi064 ceph-mon[40964]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:12.444 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:11:12.444 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (45s) 41s ago 6m 17.7M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (24s) 19s ago 6m 43.0M - c8b91775d855 92fb2e1fda05 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (5m) 41s ago 5m 17.4M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (5m) 19s ago 5m 17.4M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (108s) 19s ago 10m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (2m) 41s ago 12m 511M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (12m) 41s ago 12m 51.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 b00d59177667 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (11m) 19s ago 11m 45.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (10m) 41s ago 10m 43.4M 2048M 18.2.4-719-gced649f8 c870753d67c2 fbf010f54874 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (93s) 41s ago 6m 8048k - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (87s) 19s ago 6m 10.0M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (10m) 41s ago 10m 57.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:11:12.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (9m) 41s ago 9m 52.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (9m) 41s ago 9m 47.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (9m) 41s ago 9m 51.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (8m) 19s ago 8m 48.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (8m) 19s ago 8m 53.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (7m) 19s ago 7m 57.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (7m) 19s ago 7m 54.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:11:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (64s) 19s ago 7m 34.2M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:11:12.793 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:11:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T17:11:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:11:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:11:12.795 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "0/3 daemons upgraded", 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:11:13.106 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:11:13.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='client.? 172.21.15.64:0/1409490883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='client.? 172.21.15.64:0/1409490883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:13.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:11:13.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='client.25706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='client.25712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1409490883' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:11:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:13 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: Upgrade: Setting container_image for all mgr 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[30574]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:14.280 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:11:13] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.51.0" 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: Upgrade: Setting container_image for all mgr 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:14.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:14 smithi064 ceph-mon[40964]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all mgr 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:14 smithi175 ceph-mon[34606]: from='client.25724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: Upgrade: Updating mon.a 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:15 smithi175 ceph-mon[34606]: Deploying daemon mon.a on smithi064 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: Upgrade: Updating mon.a 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[40964]: Deploying daemon mon.a on smithi064 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: Upgrade: Updating mon.a 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: from='mgr.25556 ' entity='mgr.y' 2024-08-17T17:11:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:11:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:11:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: from='mgr.25556 172.21.15.64:0/2151830657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:11:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:15 smithi064 ceph-mon[30574]: Deploying daemon mon.a on smithi064 2024-08-17T17:11:16.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 systemd[1]: Stopping Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:11:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[30570]: 2024-08-17T17:11:16.299+0000 7ff205b7c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:11:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[30570]: 2024-08-17T17:11:16.299+0000 7ff205b7c640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-17T17:11:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 podman[104355]: 2024-08-17 17:11:16.434217969 +0000 UTC m=+0.248540383 container died b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:11:17.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 podman[104355]: 2024-08-17 17:11:16.908088865 +0000 UTC m=+0.722411258 container cleanup b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-08-17T17:11:17.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:16 smithi064 bash[104355]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:11:17.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 podman[104368]: 2024-08-17 17:11:17.142835416 +0000 UTC m=+0.709567698 container remove b00d59177667330ee46c61bb1bd373d78d0a3395fee54ae9cf64503f49af7018 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-08-17T17:11:17.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T17:11:17.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 systemd[1]: Stopped Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:11:17.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service: Consumed 14.486s CPU time. 2024-08-17T17:11:17.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 systemd[1]: Starting Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:11:17.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 podman[104497]: 2024-08-17 17:11:17.676987931 +0000 UTC m=+0.103040474 container create 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:11:17.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 podman[104497]: 2024-08-17 17:11:17.591745678 +0000 UTC m=+0.017798244 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 podman[104497]: 2024-08-17 17:11:17.888063987 +0000 UTC m=+0.314116531 container init 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 podman[104497]: 2024-08-17 17:11:17.891447745 +0000 UTC m=+0.317500291 container start 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-cd8bdbe) 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 2 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: pidfile_write: ignore empty --pid-file 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: load: jerasure load: lrc 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:11:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Git sha 0 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: DB SUMMARY 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: DB Session ID: 9SPD64DK6E38834RZPWK 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1518 Bytes 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 1940125 ; 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:11:18.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.env: 0x55d805e7abe0 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.info_log: 0x55d807585700 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.statistics: (nil) 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.use_fsync: 0 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:11:18.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.db_log_dir: 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.wal_dir: 2024-08-17T17:11:18.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.write_buffer_manager: 0x55d807588960 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:11:18.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.unordered_write: 0 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.row_cache: None 2024-08-17T17:11:18.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.wal_filter: None 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.wal_compression: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:11:18.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:11:18.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_open_files: -1 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:11:18.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Compression algorithms supported: 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kZSTD supported: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:11:18.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.merge_operator: 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_filter: None 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d807585380) 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:11:18.199 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_type: 0 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_index_type: 0 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_shortening: 1 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: checksum: 4 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: no_block_cache: 0 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache: 0x55d807579610 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_options: 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: capacity : 536870912 2024-08-17T17:11:18.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_shard_bits : 4 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: strict_capacity_limit : 0 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_compressed: (nil) 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: persistent_cache: (nil) 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size: 4096 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size_deviation: 10 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_restart_interval: 16 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_block_restart_interval: 1 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_block_size: 4096 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: partition_filters: 0 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: use_delta_encoding: 1 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: filter_policy: bloomfilter 2024-08-17T17:11:18.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: whole_key_filtering: 1 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: verify_compression: 0 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: format_version: 5 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_index_compression: 1 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_align: 0 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_auto_readahead_size: 262144 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: prepopulate_block_cache: 0 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression: NoCompression 2024-08-17T17:11:18.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.num_levels: 7 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:11:18.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:11:18.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:11:18.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:11:18.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:11:18.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.ttl: 2592000 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:11:18.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12276, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3bee3356-7a33-4cca-9d5e-d743da59347c 2024-08-17T17:11:18.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914677918916, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914677925350, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 1711954, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12277, "largest_seqno": 13690, "table_properties": {"data_size": 1705253, "index_size": 3996, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1669, "raw_key_size": 15755, "raw_average_key_size": 23, "raw_value_size": 1691311, "raw_average_value_size": 2574, "num_data_blocks": 184, "num_entries": 657, "num_filter_entries": 657, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914677, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3bee3356-7a33-4cca-9d5e-d743da59347c", "db_session_id": "9SPD64DK6E38834RZPWK", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914677925465, "job": 1, "event": "recovery_finished"} 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d8075a8e00 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: DB pointer 0x55d8076a6000 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:11:18.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** DB Stats ** 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:11:18.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 1/0 1.63 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 521.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: L6 1/0 7.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 9.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 521.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 521.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 521.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:11:18.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 182.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 182.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache BinnedLRUCache@0x55d807579610#2 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???) e3 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:18.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).mds e8 new map 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).mds e8 print_map 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: e8 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: legacy client fscid: 1 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: Filesystem 'foo' (1) 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: fs_name foo 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: epoch 7 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: created 2024-08-17T17:05:34.074034+0000 2024-08-17T17:11:18.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: modified 2024-08-17T17:05:47.828978+0000 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: tableserver 0 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: root 0 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_timeout 60 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_autoclose 300 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_file_size 1099511627776 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_xattr_size 65536 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: required_client_features {} 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure 0 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure_osd_epoch 0 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_mds 1 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: in 0 2024-08-17T17:11:18.215 INFO:journalctl@ceph.mon.a.smithi064.stdout: up {0=24554} 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: failed 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: damaged 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: stopped 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_pools [4] 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_pool 3 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: inline_data disabled 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: balancer 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: bal_rank_mask -1 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: standby_count_wanted 1 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi064.qahkwa{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.216 INFO:journalctl@ceph.mon.a.smithi064.stdout: Standby daemons: 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi175.nhcgie{-1:24508} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 ceph-mon[104516]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 bash[104497]: 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e 2024-08-17T17:11:18.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:17 smithi064 systemd[1]: Started Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: ignoring --setuser ceph since I am not root 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: ignoring --setgroup ceph since I am not root 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mgr[79095]: -- 172.21.15.175:0/769859401 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c2dbda70e0 con 0x55c2dbd84800 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: mon.a calling monitor election 2024-08-17T17:11:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: monmap epoch 3 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: min_mon_release 18 (reef) 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: election_strategy: 1 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T17:11:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: overall HEALTH_OK 2024-08-17T17:11:19.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: from='mgr.25556 ' entity='' 2024-08-17T17:11:19.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:19 smithi175 ceph-mon[34606]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T17:11:19.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ignoring --setuser ceph since I am not root 2024-08-17T17:11:19.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ignoring --setgroup ceph since I am not root 2024-08-17T17:11:19.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mgr[86913]: -- 172.21.15.64:0/3087537908 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5650908f50e0 con 0x5650908d3000 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: mon.a calling monitor election 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: monmap epoch 3 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: min_mon_release 18 (reef) 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: election_strategy: 1 2024-08-17T17:11:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: overall HEALTH_OK 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: from='mgr.25556 ' entity='' 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[104516]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: mon.a calling monitor election 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:19.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: monmap epoch 3 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: min_mon_release 18 (reef) 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: election_strategy: 1 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T17:11:19.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: overall HEALTH_OK 2024-08-17T17:11:19.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: from='mgr.25556 ' entity='' 2024-08-17T17:11:19.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:19 smithi064 ceph-mon[40964]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T17:11:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: Standby manager daemon x restarted 2024-08-17T17:11:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: Standby manager daemon x started 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-mon[34606]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:11:24] ENGINE Bus STARTING 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: CherryPy Checker: 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: The Application mounted at '' has an empty config. 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:11:24] ENGINE Serving on http://:::9283 2024-08-17T17:11:24.662 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:11:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x[79091]: [17/Aug/2024:17:11:24] ENGINE Bus STARTED 2024-08-17T17:11:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: Standby manager daemon x restarted 2024-08-17T17:11:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: Standby manager daemon x started 2024-08-17T17:11:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[104516]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: Standby manager daemon x restarted 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: Standby manager daemon x started 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:11:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:24 smithi064 ceph-mon[40964]: from='mgr.? 172.21.15.175:0/150225003' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:11:25.319 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.058+0000 7f06370d8640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.319 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.058+0000 7f06370d8640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.058+0000 7f06370d8640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.058+0000 7f06370d8640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.058+0000 7f06370d8640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.059+0000 7f063a0de640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.059+0000 7f063a0de640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.059+0000 7f063a0de640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.059+0000 7f063a0de640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.059+0000 7f063a0de640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:11:25.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: Active manager daemon y restarted 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: Activating manager daemon y 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: mgrmap e37: y(active, starting, since 0.0067474s), standbys: x 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"} : dispatch 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"} : dispatch 2024-08-17T17:11:25.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:11:25.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: Manager daemon y is now available 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: Active manager daemon y restarted 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: Activating manager daemon y 2024-08-17T17:11:25.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: mgrmap e37: y(active, starting, since 0.0067474s), standbys: x 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:11:25.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: Manager daemon y is now available 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:11:25.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:11:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-17T17:11:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: Active manager daemon y restarted 2024-08-17T17:11:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: Activating manager daemon y 2024-08-17T17:11:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: mgrmap e37: y(active, starting, since 0.0067474s), standbys: x 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi175.nhcgie"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi064.qahkwa"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:11:25.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: Manager daemon y is now available 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:11:25.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:11:25.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.576+0000 7f06693d0640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-17T17:11:25.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:11:25] ENGINE Bus STARTING 2024-08-17T17:11:25.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: CherryPy Checker: 2024-08-17T17:11:25.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: The Application mounted at '' has an empty config. 2024-08-17T17:11:25.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:25.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:11:25] ENGINE Serving on http://:::9283 2024-08-17T17:11:25.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:11:25] ENGINE Bus STARTED 2024-08-17T17:11:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:26 smithi175 ceph-mon[34606]: mgrmap e38: y(active, since 1.01392s), standbys: x 2024-08-17T17:11:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:26 smithi175 ceph-mon[34606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:11:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:26 smithi175 ceph-mon[34606]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[104516]: mgrmap e38: y(active, since 1.01392s), standbys: x 2024-08-17T17:11:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:11:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[104516]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[40964]: mgrmap e38: y(active, since 1.01392s), standbys: x 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[40964]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-mon[40964]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:26.666+0000 7f06176df640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:11:26] ENGINE Error in HTTPServer.serve 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: Traceback (most recent call last): 2024-08-17T17:11:26.938 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._connections.run(self.expiration_interval) 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._run(expiration_interval) 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: s = self.context.wrap_socket( 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: return self.sslsocket_class._create( 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:11:26.939 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self.do_handshake() 2024-08-17T17:11:26.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:11:26.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: self._sslobj.do_handshake() 2024-08-17T17:11:26.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:11:26.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:26 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:27.861 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: [17/Aug/2024:17:11:26] ENGINE Bus STARTING 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: [17/Aug/2024:17:11:26] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: [17/Aug/2024:17:11:26] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: [17/Aug/2024:17:11:26] ENGINE Bus STARTED 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:27 smithi175 ceph-mon[34606]: [17/Aug/2024:17:11:26] ENGINE Error in HTTPServer.serve 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout: Traceback (most recent call last): 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:11:27.862 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._run(expiration_interval) 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: s = self.context.wrap_socket( 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: return self.sslsocket_class._create( 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: self.do_handshake() 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:11:27.863 INFO:journalctl@ceph.mon.b.smithi175.stdout: self._sslobj.do_handshake() 2024-08-17T17:11:27.864 INFO:journalctl@ceph.mon.b.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: [17/Aug/2024:17:11:26] ENGINE Bus STARTING 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: [17/Aug/2024:17:11:26] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: [17/Aug/2024:17:11:26] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: [17/Aug/2024:17:11:26] ENGINE Bus STARTED 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[104516]: [17/Aug/2024:17:11:26] ENGINE Error in HTTPServer.serve 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:11:27.891 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: self.do_handshake() 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:11:27.892 INFO:journalctl@ceph.mon.a.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.a.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: [17/Aug/2024:17:11:26] ENGINE Bus STARTING 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: [17/Aug/2024:17:11:26] ENGINE Serving on http://172.21.15.64:8765 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: [17/Aug/2024:17:11:26] ENGINE Serving on https://172.21.15.64:7150 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: [17/Aug/2024:17:11:26] ENGINE Bus STARTED 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:27 smithi064 ceph-mon[40964]: [17/Aug/2024:17:11:26] ENGINE Error in HTTPServer.serve 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout: Traceback (most recent call last): 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:11:27.893 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._run(expiration_interval) 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: s = self.context.wrap_socket( 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: return self.sslsocket_class._create( 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:11:27.894 INFO:journalctl@ceph.mon.c.smithi064.stdout: self.do_handshake() 2024-08-17T17:11:27.895 INFO:journalctl@ceph.mon.c.smithi064.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:11:27.895 INFO:journalctl@ceph.mon.c.smithi064.stdout: self._sslobj.do_handshake() 2024-08-17T17:11:27.895 INFO:journalctl@ceph.mon.c.smithi064.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:29 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:11:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:29.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:29 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:31 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:31 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:31 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:31 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:32.428 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:32 smithi175 ceph-mon[34606]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:32 smithi064 ceph-mon[104516]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:32.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:32 smithi064 ceph-mon[40964]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:11:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:33 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:11:33] "GET /metrics HTTP/1.1" 200 31340 "" "Prometheus/2.51.0" 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:11:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: Updating smithi064:/etc/ceph/ceph.conf 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: Updating smithi175:/etc/ceph/ceph.conf 2024-08-17T17:11:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:33 smithi064 ceph-mon[104516]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[40964]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:11:35.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:35 smithi064 ceph-mon[104516]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Updating smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Updating smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:11:35.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:35 smithi175 ceph-mon[34606]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:11:36.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[104516]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[40964]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:11:36.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:11:36.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:36 smithi064 ceph-mon[40964]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:36 smithi175 ceph-mon[34606]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:11:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:36 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:36 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:11:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:36 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:11:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:36 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:37.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-mon[104516]: Upgrade: Updating mon.c 2024-08-17T17:11:37.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-mon[104516]: Deploying daemon mon.c on smithi064 2024-08-17T17:11:37.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-mon[40964]: Upgrade: Updating mon.c 2024-08-17T17:11:37.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-mon[40964]: Deploying daemon mon.c on smithi064 2024-08-17T17:11:37.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 systemd[1]: Stopping Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:11:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:37 smithi175 ceph-mon[34606]: Upgrade: Updating mon.c 2024-08-17T17:11:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:37 smithi175 ceph-mon[34606]: Deploying daemon mon.c on smithi064 2024-08-17T17:11:37.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c[40960]: 2024-08-17T17:11:37.648+0000 7faeef42c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:11:37.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c[40960]: 2024-08-17T17:11:37.648+0000 7faeef42c640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:11:37.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:37 smithi064 podman[109073]: 2024-08-17 17:11:37.894241013 +0000 UTC m=+0.367282135 container died fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:11:38.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 podman[109073]: 2024-08-17 17:11:38.399247162 +0000 UTC m=+0.872288280 container cleanup fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:11:38.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 bash[109073]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:11:38.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 podman[109088]: 2024-08-17 17:11:38.650647307 +0000 UTC m=+0.748586523 container remove fbf010f54874be351b3f8525ed2cc3ed64a15a85213962bba051566b9a0a9d8a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:11:38.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-17T17:11:38.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 systemd[1]: Stopped Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:11:38.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:38 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.c.service: Consumed 8.157s CPU time. 2024-08-17T17:11:39.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 systemd[1]: Starting Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:11:39.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 podman[109225]: 2024-08-17 17:11:39.161068471 +0000 UTC m=+0.018087998 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 podman[109225]: 2024-08-17 17:11:39.285056832 +0000 UTC m=+0.142076349 container create 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 podman[109225]: 2024-08-17 17:11:39.487755535 +0000 UTC m=+0.344775053 container init 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe) 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 podman[109225]: 2024-08-17 17:11:39.491232531 +0000 UTC m=+0.348252058 container start 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe) 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 2 2024-08-17T17:11:39.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: pidfile_write: ignore empty --pid-file 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: load: jerasure load: lrc 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Git sha 0 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: DB SUMMARY 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: DB Session ID: ZLII7UWITC7JEGE8WXS6 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 5977816 ; 2024-08-17T17:11:39.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:11:39.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:11:39.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:11:39.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.env: 0x56477af44be0 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.info_log: 0x56477d525700 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.statistics: (nil) 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.use_fsync: 0 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:11:39.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.db_log_dir: 2024-08-17T17:11:39.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.wal_dir: 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.write_buffer_manager: 0x56477d528960 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:11:39.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.unordered_write: 0 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:11:39.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.row_cache: None 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.wal_filter: None 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.wal_compression: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:11:39.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:11:39.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_open_files: -1 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:11:39.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Compression algorithms supported: 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kZSTD supported: 0 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-08-17T17:11:39.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.merge_operator: 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_filter: None 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56477d525380) 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:11:39.548 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_type: 0 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_index_type: 0 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_shortening: 1 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: checksum: 4 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: no_block_cache: 0 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache: 0x56477d519610 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_options: 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: capacity : 536870912 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_shard_bits : 4 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: strict_capacity_limit : 0 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:11:39.549 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_compressed: (nil) 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: persistent_cache: (nil) 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size: 4096 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size_deviation: 10 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_restart_interval: 16 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_block_restart_interval: 1 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_block_size: 4096 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: partition_filters: 0 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: use_delta_encoding: 1 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: filter_policy: bloomfilter 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: whole_key_filtering: 1 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: verify_compression: 0 2024-08-17T17:11:39.550 INFO:journalctl@ceph.mon.c.smithi064.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: format_version: 5 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_index_compression: 1 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_align: 0 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_auto_readahead_size: 262144 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: prepopulate_block_cache: 0 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression: NoCompression 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.num_levels: 7 2024-08-17T17:11:39.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:11:39.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:11:39.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:11:39.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:11:39.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:11:39.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.ttl: 2592000 2024-08-17T17:11:39.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:11:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12869, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 609b92da-c0bc-488a-b55a-26ffc658d8d9 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914699518634, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914699534267, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3654322, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12874, "largest_seqno": 14915, "table_properties": {"data_size": 3645511, "index_size": 5589, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2181, "raw_key_size": 20680, "raw_average_key_size": 23, "raw_value_size": 3626987, "raw_average_value_size": 4178, "num_data_blocks": 255, "num_entries": 868, "num_filter_entries": 868, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914699, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "609b92da-c0bc-488a-b55a-26ffc658d8d9", "db_session_id": "ZLII7UWITC7JEGE8WXS6", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914699534399, "job": 1, "event": "recovery_finished"} 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:11:39.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56477d548e00 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: DB pointer 0x56477d646000 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** DB Stats ** 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:11:39.560 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 3.49 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.1 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: L6 1/0 7.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 2/0 11.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.1 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.1 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 497.1 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.561 INFO:journalctl@ceph.mon.c.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.00 GB write, 188.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.00 GB write, 188.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache BinnedLRUCache@0x56477d519610#2 capacity: 512.00 MB usage: 40.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache entry stats(count,size,portion): DataBlock(1,6.58 KB,0.00125468%) FilterBlock(2,11.31 KB,0.00215769%) IndexBlock(2,22.16 KB,0.00422597%) Misc(1,0.00 KB,0%) 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.562 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: starting mon.c rank 2 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???) e3 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).mds e8 new map 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).mds e8 print_map 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: e8 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: legacy client fscid: 1 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: Filesystem 'foo' (1) 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: fs_name foo 2024-08-17T17:11:39.563 INFO:journalctl@ceph.mon.c.smithi064.stdout: epoch 7 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: created 2024-08-17T17:05:34.074034+0000 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: modified 2024-08-17T17:05:47.828978+0000 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: tableserver 0 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: root 0 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_timeout 60 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_autoclose 300 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_file_size 1099511627776 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_xattr_size 65536 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: required_client_features {} 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure 0 2024-08-17T17:11:39.564 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure_osd_epoch 0 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_mds 1 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: in 0 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: up {0=24554} 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: failed 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: damaged 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: stopped 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_pools [4] 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_pool 3 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: inline_data disabled 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: balancer 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: bal_rank_mask -1 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: standby_count_wanted 1 2024-08-17T17:11:39.565 INFO:journalctl@ceph.mon.c.smithi064.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi064.qahkwa{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: Standby daemons: 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi175.nhcgie{-1:24508} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:11:39.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 ceph-mon[109246]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-08-17T17:11:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 bash[109225]: 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 2024-08-17T17:11:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:39 smithi064 systemd[1]: Started Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:11:44.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:11:43] "GET /metrics HTTP/1.1" 200 31340 "" "Prometheus/2.51.0" 2024-08-17T17:11:45.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:44 smithi175 ceph-mon[34606]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:11:45.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:44 smithi064 ceph-mon[104516]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: mon.c calling monitor election 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: mon.a calling monitor election 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: mon.b calling monitor election 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: monmap epoch 3 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:46.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: min_mon_release 18 (reef) 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: election_strategy: 1 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: mgrmap e40: y(active, since 21s), standbys: x 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[109246]: overall HEALTH_OK 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: mon.c calling monitor election 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: mon.a calling monitor election 2024-08-17T17:11:46.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: mon.b calling monitor election 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: monmap epoch 3 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: min_mon_release 18 (reef) 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: election_strategy: 1 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: mgrmap e40: y(active, since 21s), standbys: x 2024-08-17T17:11:46.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:45 smithi064 ceph-mon[104516]: overall HEALTH_OK 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: mon.c calling monitor election 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: mon.a calling monitor election 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: mon.b calling monitor election 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: monmap epoch 3 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: last_changed 2024-08-17T17:00:19.343721+0000 2024-08-17T17:11:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: min_mon_release 18 (reef) 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: election_strategy: 1 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: mgrmap e40: y(active, since 21s), standbys: x 2024-08-17T17:11:46.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:45 smithi175 ceph-mon[34606]: overall HEALTH_OK 2024-08-17T17:11:46.433 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:11:47.043 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:11:47.043 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (80s) 18s ago 7m 20.1M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (58s) 17s ago 7m 75.0M - c8b91775d855 92fb2e1fda05 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (6m) 18s ago 6m 17.9M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (6m) 17s ago 6m 17.7M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (2m) 17s ago 11m 450M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (2m) 18s ago 12m 487M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (29s) 18s ago 13m 30.3M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (11m) 17s ago 11m 51.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 starting - - - 2048M 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (2m) 18s ago 7m 11.4M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (2m) 17s ago 7m 11.7M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (10m) 18s ago 10m 57.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (10m) 18s ago 10m 53.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:11:47.044 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (10m) 18s ago 10m 47.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (9m) 18s ago 9m 51.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (9m) 17s ago 9m 48.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (8m) 17s ago 8m 54.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (8m) 17s ago 8m 57.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (8m) 17s ago 8m 54.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:11:47.045 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (99s) 17s ago 7m 41.5M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:11:47.087 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:46 smithi064 ceph-mon[104516]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.087 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:46 smithi064 ceph-mon[109246]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:46 smithi175 ceph-mon[34606]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.396 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:11:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:11:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:11:47.397 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:11:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 11, 2024-08-17T17:11:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:11:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:11:47.398 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "1/3 daemons upgraded", 2024-08-17T17:11:47.705 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:11:47.706 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:11:47.706 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:11:47.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[109246]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[109246]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[109246]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1298820319' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[104516]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[104516]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[104516]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:47.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:47 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1298820319' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:48.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:47 smithi175 ceph-mon[34606]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:48.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:47 smithi175 ceph-mon[34606]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:48.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:47 smithi175 ceph-mon[34606]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:48.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:47 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/1298820319' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[104516]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[109246]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:49 smithi175 ceph-mon[34606]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:49 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:49 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:49 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:49 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:49.687 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:11:49.547+0000 7f06693d0640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-17T17:11:50.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:50 smithi175 ceph-mon[34606]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:50 smithi064 ceph-mon[104516]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:50.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:50 smithi064 ceph-mon[109246]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:52.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:52.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:11:52.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:52.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:11:52.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:52.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:52 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all grafana 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all loki 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all promtail 2024-08-17T17:11:53.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Finalizing container_image settings 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:11:53.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:11:53.415 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: Upgrade: Complete! 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.416 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.417 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all grafana 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all loki 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all promtail 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Finalizing container_image settings 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:11:53.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all grafana 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all loki 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all promtail 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Finalizing container_image settings 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:11:53.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: Upgrade: Complete! 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:11:53.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: Upgrade: Complete! 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:53.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:53.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:53.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:53.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:54.300 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:11:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:11:53] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.51.0" 2024-08-17T17:11:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:54 smithi175 ceph-mon[34606]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:54 smithi064 ceph-mon[104516]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:54 smithi064 ceph-mon[109246]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:55 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:55 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:11:56.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:56 smithi175 ceph-mon[34606]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:56 smithi064 ceph-mon[104516]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:56 smithi064 ceph-mon[109246]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:11:58 smithi175 ceph-mon[34606]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:11:58 smithi064 ceph-mon[104516]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:11:58.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:11:58 smithi064 ceph-mon[109246]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:00 smithi175 ceph-mon[34606]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:00 smithi064 ceph-mon[104516]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:00 smithi064 ceph-mon[109246]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:02 smithi175 ceph-mon[34606]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:02 smithi064 ceph-mon[104516]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:02.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:02 smithi064 ceph-mon[109246]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:04.326 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:03] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.51.0" 2024-08-17T17:12:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:04 smithi175 ceph-mon[34606]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:04 smithi064 ceph-mon[104516]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:04.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:04 smithi064 ceph-mon[109246]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:06 smithi175 ceph-mon[34606]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:06 smithi064 ceph-mon[104516]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:06.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:06 smithi064 ceph-mon[109246]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:08 smithi175 ceph-mon[34606]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:08.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:08 smithi064 ceph-mon[109246]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:08.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:08 smithi064 ceph-mon[104516]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:10 smithi175 ceph-mon[34606]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:10 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:10 smithi064 ceph-mon[109246]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:10 smithi064 ceph-mon[104516]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:12 smithi175 ceph-mon[34606]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:12.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:12 smithi064 ceph-mon[109246]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:12 smithi064 ceph-mon[104516]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:14.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:12:13 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=infra.usagestats t=2024-08-17T17:12:13.892814775Z level=info msg="Usage stats are ready to report" 2024-08-17T17:12:14.340 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:13] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.51.0" 2024-08-17T17:12:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:14 smithi175 ceph-mon[34606]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:14.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:14 smithi064 ceph-mon[109246]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:14 smithi064 ceph-mon[104516]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:16 smithi175 ceph-mon[34606]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:16.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:16 smithi064 ceph-mon[109246]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:16 smithi064 ceph-mon[104516]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:18 smithi175 ceph-mon[34606]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:18 smithi064 ceph-mon[104516]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:18.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:18 smithi064 ceph-mon[109246]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:18.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:12:19.033 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:19 smithi064 ceph-mon[104516]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:19.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:19 smithi064 ceph-mon[109246]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:19 smithi175 ceph-mon[34606]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:20 smithi175 ceph-mon[34606]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:20 smithi064 ceph-mon[104516]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:20 smithi064 ceph-mon[109246]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:22 smithi175 ceph-mon[34606]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:22 smithi064 ceph-mon[104516]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:22 smithi064 ceph-mon[109246]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:24.358 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:23] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.51.0" 2024-08-17T17:12:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:24 smithi175 ceph-mon[34606]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:24 smithi064 ceph-mon[104516]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:24 smithi064 ceph-mon[109246]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:26 smithi175 ceph-mon[34606]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:26 smithi064 ceph-mon[104516]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:26.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:26 smithi064 ceph-mon[109246]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:28 smithi175 ceph-mon[34606]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:28 smithi064 ceph-mon[104516]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:28 smithi064 ceph-mon[109246]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:30 smithi175 ceph-mon[34606]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:30 smithi064 ceph-mon[104516]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:30 smithi064 ceph-mon[109246]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:32 smithi175 ceph-mon[34606]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:32 smithi064 ceph-mon[104516]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:32 smithi064 ceph-mon[109246]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:34.376 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:33] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.51.0" 2024-08-17T17:12:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:34 smithi175 ceph-mon[34606]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:34 smithi064 ceph-mon[104516]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:34 smithi064 ceph-mon[109246]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:36 smithi175 ceph-mon[34606]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:36 smithi064 ceph-mon[104516]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:36 smithi064 ceph-mon[109246]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:38 smithi175 ceph-mon[34606]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:38 smithi064 ceph-mon[104516]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:38 smithi064 ceph-mon[109246]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:40 smithi175 ceph-mon[34606]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:40 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:40 smithi064 ceph-mon[104516]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:40 smithi064 ceph-mon[109246]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:42 smithi175 ceph-mon[34606]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:42 smithi064 ceph-mon[104516]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:42.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:42 smithi064 ceph-mon[109246]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:44.392 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:43] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.51.0" 2024-08-17T17:12:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:44 smithi175 ceph-mon[34606]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:44 smithi064 ceph-mon[104516]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:44 smithi064 ceph-mon[109246]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:46 smithi175 ceph-mon[34606]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:46 smithi064 ceph-mon[104516]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:46.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:46 smithi064 ceph-mon[109246]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:48 smithi175 ceph-mon[34606]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:48 smithi064 ceph-mon[104516]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:48 smithi064 ceph-mon[109246]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:50 smithi175 ceph-mon[34606]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:50 smithi064 ceph-mon[104516]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:50 smithi064 ceph-mon[109246]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:52 smithi175 ceph-mon[34606]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:52 smithi064 ceph-mon[104516]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:52.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:52 smithi064 ceph-mon[109246]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:53.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:53.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:53 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:12:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:12:53] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.51.0" 2024-08-17T17:12:54.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:54 smithi064 ceph-mon[104516]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:54 smithi064 ceph-mon[109246]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:54.749 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:54 smithi175 ceph-mon[34606]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:55.679 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:55 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:55.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[104516]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[109246]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:56.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:56 smithi175 ceph-mon[34606]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:56 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:56 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:57 smithi175 ceph-mon[34606]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:12:57.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[104516]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:12:57.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:57 smithi064 ceph-mon[109246]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:12:59.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:12:59 smithi175 ceph-mon[34606]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:12:59.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:12:59 smithi064 ceph-mon[104516]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:12:59.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:12:59 smithi064 ceph-mon[109246]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:01 smithi175 ceph-mon[34606]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:01 smithi064 ceph-mon[104516]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:01 smithi064 ceph-mon[109246]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:03.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:03 smithi175 ceph-mon[34606]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:03 smithi064 ceph-mon[104516]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:03.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:03 smithi064 ceph-mon[109246]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:04.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:03] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.51.0" 2024-08-17T17:13:05.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:05 smithi175 ceph-mon[34606]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:05.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:05 smithi064 ceph-mon[104516]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:05 smithi064 ceph-mon[109246]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:07 smithi175 ceph-mon[34606]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:07.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:07 smithi064 ceph-mon[104516]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:07.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:07 smithi064 ceph-mon[109246]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:13:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:09 smithi175 ceph-mon[34606]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:09 smithi064 ceph-mon[104516]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:09 smithi064 ceph-mon[109246]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:10 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:11 smithi175 ceph-mon[34606]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:11.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:11 smithi064 ceph-mon[104516]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:11.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:11 smithi064 ceph-mon[109246]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:13 smithi175 ceph-mon[34606]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:13 smithi064 ceph-mon[104516]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:13 smithi064 ceph-mon[109246]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:14.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:13] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-08-17T17:13:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:15 smithi175 ceph-mon[34606]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:15.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:15 smithi064 ceph-mon[104516]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:15.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:15 smithi064 ceph-mon[109246]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:17 smithi175 ceph-mon[34606]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:17 smithi064 ceph-mon[104516]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:17 smithi064 ceph-mon[109246]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:19 smithi175 ceph-mon[34606]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:19 smithi064 ceph-mon[104516]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:19 smithi064 ceph-mon[109246]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for majority of mons to upgrade' 2024-08-17T17:13:21.468 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:21.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:21 smithi064 ceph-mon[104516]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:21 smithi064 ceph-mon[109246]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:21 smithi175 ceph-mon[34606]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:22.849 INFO:teuthology.orchestra.run.smithi064.stdout:wait for majority of mons to upgrade 2024-08-17T17:13:23.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-17T17:13:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:23.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:23 smithi064 ceph-mon[104516]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:23.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:23 smithi064 ceph-mon[109246]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:23 smithi175 ceph-mon[34606]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:23] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.51.0" 2024-08-17T17:13:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:25 smithi175 ceph-mon[34606]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:25 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:25 smithi175 ceph-mon[34606]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[104516]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[104516]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:25.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[109246]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:25.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:25 smithi064 ceph-mon[109246]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:27.111 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:27.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:13:27.902 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:28.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:28.307 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[104516]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:28.309 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:28.309 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:28.309 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.309 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:28 smithi064 ceph-mon[109246]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:28.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:28 smithi175 ceph-mon[34606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:13:30.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:13:30.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:13:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:13:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:30 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (3m) 35s ago 9m 20.2M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (2m) 34s ago 8m 77.2M - c8b91775d855 92fb2e1fda05 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (7m) 35s ago 7m 18.5M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (7m) 34s ago 7m 18.0M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (4m) 34s ago 12m 453M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:13:31.309 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (4m) 35s ago 14m 508M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (2m) 35s ago 14m 40.7M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (13m) 34s ago 13m 55.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 8704821b63cd 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (112s) 35s ago 13m 32.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (3m) 35s ago 9m 10.9M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (3m) 34s ago 8m 11.0M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (12m) 35s ago 12m 58.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (12m) 35s ago 12m 53.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (11m) 35s ago 11m 47.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (11m) 35s ago 11m 52.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (11m) 34s ago 11m 49.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (10m) 34s ago 10m 54.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:13:31.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (10m) 34s ago 10m 58.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:13:31.311 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (9m) 34s ago 9m 55.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:13:31.311 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (3m) 34s ago 9m 45.8M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:13:31.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:31 smithi175 ceph-mon[34606]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:13:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:31 smithi175 ceph-mon[34606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:13:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:31 smithi175 ceph-mon[34606]: Upgrade: Setting container_image for all mgr 2024-08-17T17:13:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:31 smithi175 ceph-mon[34606]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:13:31.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[104516]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:13:31.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[104516]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all mgr 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[104516]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[109246]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[109246]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all mgr 2024-08-17T17:13:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:31 smithi064 ceph-mon[109246]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:13:31.664 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:13:31.664 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:13:31.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:13:31.666 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:13:31.666 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 11, 2024-08-17T17:13:31.666 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:13:31.666 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:13:31.666 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "2/3 daemons upgraded", 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:13:31.964 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:32 smithi175 ceph-mon[34606]: from='client.? 172.21.15.64:0/4195006822' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4195006822' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:32.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4195006822' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:13:33.123 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 systemd[1]: Stopping Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:13:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 ceph-mon[34606]: Upgrade: Updating mon.b 2024-08-17T17:13:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 ceph-mon[34606]: Deploying daemon mon.b on smithi175 2024-08-17T17:13:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 ceph-mon[34606]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b[34602]: 2024-08-17T17:13:33.306+0000 7f57717ec640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:13:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b[34602]: 2024-08-17T17:13:33.306+0000 7f57717ec640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:13:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[104516]: Upgrade: Updating mon.b 2024-08-17T17:13:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[104516]: Deploying daemon mon.b on smithi175 2024-08-17T17:13:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[104516]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[109246]: Upgrade: Updating mon.b 2024-08-17T17:13:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[109246]: Deploying daemon mon.b on smithi175 2024-08-17T17:13:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-mon[109246]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:13:33.889 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 podman[101274]: 2024-08-17 17:13:33.541804767 +0000 UTC m=+0.356884335 container died 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, io.buildah.version=1.36.0, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:13:34.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 podman[101274]: 2024-08-17 17:13:33.889689059 +0000 UTC m=+0.704768627 container cleanup 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:13:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:33 smithi175 bash[101274]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:13:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:33] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.51.0" 2024-08-17T17:13:34.513 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 podman[101286]: 2024-08-17 17:13:34.191291942 +0000 UTC m=+0.648899715 container remove 8704821b63cd164af428b881bd64d7546138f60e3eca8b1ac1b0db8207451956 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-ced649f) 2024-08-17T17:13:34.513 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:13:34.513 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 systemd[1]: Stopped Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:13:34.513 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service: Consumed 9.698s CPU time. 2024-08-17T17:13:34.818 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 systemd[1]: Starting Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:13:34.818 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 podman[101415]: 2024-08-17 17:13:34.725297476 +0000 UTC m=+0.109670912 container create 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:13:34.818 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 podman[101415]: 2024-08-17 17:13:34.633286954 +0000 UTC m=+0.017660408 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:13:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 podman[101415]: 2024-08-17 17:13:34.91974338 +0000 UTC m=+0.304116816 container init 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 podman[101415]: 2024-08-17 17:13:34.922836319 +0000 UTC m=+0.307209758 container start 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 2 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: pidfile_write: ignore empty --pid-file 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: load: jerasure load: lrc 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Git sha 0 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: DB SUMMARY 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: DB Session ID: PEG44UC3TA15LAGBY2B5 2024-08-17T17:13:35.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 8062068 ; 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:13:35.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.env: 0x55e8dcd2dbe0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.info_log: 0x55e8dec59700 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.statistics: (nil) 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.use_fsync: 0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:13:35.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.db_log_dir: 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.wal_dir: 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:13:35.165 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.write_buffer_manager: 0x55e8dec5c960 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.unordered_write: 0 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:13:35.166 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:13:35.167 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:13:35.167 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.row_cache: None 2024-08-17T17:13:35.167 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.wal_filter: None 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.wal_compression: 0 2024-08-17T17:13:35.168 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:13:35.169 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_open_files: -1 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:13:35.170 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Compression algorithms supported: 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kZSTD supported: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:13:35.171 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.merge_operator: 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_filter: None 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e8dec59380) 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:13:35.172 INFO:journalctl@ceph.mon.b.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_type: 0 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: data_block_index_type: 0 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_shortening: 1 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: checksum: 4 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: no_block_cache: 0 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache: 0x55e8dec4d610 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_options: 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: capacity : 536870912 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: num_shard_bits : 4 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: strict_capacity_limit : 0 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_cache_compressed: (nil) 2024-08-17T17:13:35.173 INFO:journalctl@ceph.mon.b.smithi175.stdout: persistent_cache: (nil) 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_size: 4096 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_size_deviation: 10 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_restart_interval: 16 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: index_block_restart_interval: 1 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: metadata_block_size: 4096 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: partition_filters: 0 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: use_delta_encoding: 1 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: filter_policy: bloomfilter 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: whole_key_filtering: 1 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: verify_compression: 0 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: format_version: 5 2024-08-17T17:13:35.174 INFO:journalctl@ceph.mon.b.smithi175.stdout: enable_index_compression: 1 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout: block_align: 0 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout: max_auto_readahead_size: 262144 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout: prepopulate_block_cache: 0 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression: NoCompression 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.num_levels: 7 2024-08-17T17:13:35.175 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:13:35.176 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:13:35.177 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:13:35.178 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:13:35.179 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.ttl: 2592000 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:13:35.180 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12949, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:13:35.181 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: afd1e43a-e081-4717-9fc9-1ba4119c5ecd 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914814949623, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914814969783, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4932958, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12954, "largest_seqno": 16693, "table_properties": {"data_size": 4918543, "index_size": 9208, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 36829, "raw_average_key_size": 22, "raw_value_size": 4885871, "raw_average_value_size": 2970, "num_data_blocks": 415, "num_entries": 1645, "num_filter_entries": 1645, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914814, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "afd1e43a-e081-4717-9fc9-1ba4119c5ecd", "db_session_id": "PEG44UC3TA15LAGBY2B5", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914814969934, "job": 1, "event": "recovery_finished"} 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e8dec7ce00 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: DB pointer 0x55e8ded7a000 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** DB Stats ** 2024-08-17T17:13:35.182 INFO:journalctl@ceph.mon.b.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** Compaction Stats [default] ** 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: L0 1/0 4.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: L6 1/0 7.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Sum 2/0 12.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-08-17T17:13:35.183 INFO:journalctl@ceph.mon.b.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** Compaction Stats [default] ** 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 534.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:13:35.184 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: Cumulative compaction: 0.00 GB write, 190.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: Interval compaction: 0.00 GB write, 190.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: Block cache BinnedLRUCache@0x55e8dec4d610#2 capacity: 512.00 MB usage: 13.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: starting mon.b rank 1 at public addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] at bind addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???) e3 preinit fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).mds e8 new map 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).mds e8 print_map 2024-08-17T17:13:35.185 INFO:journalctl@ceph.mon.b.smithi175.stdout: e8 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: legacy client fscid: 1 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: Filesystem 'foo' (1) 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: fs_name foo 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: epoch 7 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: created 2024-08-17T17:05:34.074034+0000 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: modified 2024-08-17T17:05:47.828978+0000 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: tableserver 0 2024-08-17T17:13:35.186 INFO:journalctl@ceph.mon.b.smithi175.stdout: root 0 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: session_timeout 60 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: session_autoclose 300 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: max_file_size 1099511627776 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: max_xattr_size 65536 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: required_client_features {} 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: last_failure 0 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: last_failure_osd_epoch 0 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: max_mds 1 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: in 0 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: up {0=24554} 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: failed 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: damaged 2024-08-17T17:13:35.187 INFO:journalctl@ceph.mon.b.smithi175.stdout: stopped 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: data_pools [4] 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: metadata_pool 3 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: inline_data disabled 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: balancer 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: bal_rank_mask -1 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: standby_count_wanted 1 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: [mds.foo.smithi064.qahkwa{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/2165820549,v1:172.21.15.64:6835/2165820549] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: Standby daemons: 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: 2024-08-17T17:13:35.188 INFO:journalctl@ceph.mon.b.smithi175.stdout: [mds.foo.smithi175.nhcgie{-1:24508} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.175:6832/648275677,v1:172.21.15.175:6833/648275677] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:34 smithi175 ceph-mon[101434]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:35 smithi175 bash[101415]: 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 2024-08-17T17:13:35.189 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:35 smithi175 systemd[1]: Started Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:13:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:36 smithi064 ceph-mon[104516]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:36 smithi064 ceph-mon[109246]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:36 smithi175 ceph-mon[101434]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: mon.b calling monitor election 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: mon.c calling monitor election 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: mon.a calling monitor election 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: monmap epoch 4 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: last_changed 2024-08-17T17:13:36.394533+0000 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: min_mon_release 19 (squid) 2024-08-17T17:13:37.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: election_strategy: 1 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: overall HEALTH_OK 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: mon.b calling monitor election 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: mon.c calling monitor election 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: mon.a calling monitor election 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: monmap epoch 4 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:13:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: last_changed 2024-08-17T17:13:36.394533+0000 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: min_mon_release 19 (squid) 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: election_strategy: 1 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: overall HEALTH_OK 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:37.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: mon.b calling monitor election 2024-08-17T17:13:37.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:13:37.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: mon.c calling monitor election 2024-08-17T17:13:37.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:13:37.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: mon.a calling monitor election 2024-08-17T17:13:37.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:13:37.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:13:37.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: monmap epoch 4 2024-08-17T17:13:37.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:13:37.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: last_changed 2024-08-17T17:13:36.394533+0000 2024-08-17T17:13:37.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: created 2024-08-17T16:58:38.717878+0000 2024-08-17T17:13:37.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: min_mon_release 19 (squid) 2024-08-17T17:13:37.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: election_strategy: 1 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.b 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: 2: [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon.c 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: fsmap foo:1 {0=foo.smithi064.qahkwa=up:active} 1 up:standby 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: overall HEALTH_OK 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:37.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:38 smithi064 ceph-mon[104516]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:38 smithi064 ceph-mon[109246]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:38.777 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:38 smithi175 ceph-mon[101434]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:40.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:39 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:13:39.982+0000 7f06693d0640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-17T17:13:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:40 smithi175 ceph-mon[101434]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:40 smithi064 ceph-mon[104516]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:40 smithi064 ceph-mon[109246]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:41.706 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.706 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.706 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:41 smithi175 ceph-mon[101434]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:41.706 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.707 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[104516]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[109246]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:43.859 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:43 smithi175 ceph-mon[101434]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:43.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:43 smithi064 ceph-mon[104516]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:43.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:43 smithi064 ceph-mon[109246]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:43] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.51.0" 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:13:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:45.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:45.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:45 smithi175 ceph-mon[101434]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[104516]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:45.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:45.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:45.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:45 smithi064 ceph-mon[109246]: Reconfiguring daemon mon.a on smithi064 2024-08-17T17:13:46.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:46.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:47.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:47.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: Reconfiguring daemon mgr.y on smithi064 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:48.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[104516]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:48.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[104516]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:13:48.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[104516]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:13:48.430 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[109246]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:48.430 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[109246]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:13:48.430 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:48 smithi064 ceph-mon[109246]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:13:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:48 smithi175 ceph-mon[101434]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:48 smithi175 ceph-mon[101434]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:13:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:48 smithi175 ceph-mon[101434]: Reconfiguring daemon mon.c on smithi064 2024-08-17T17:13:49.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:13:49.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:49.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:13:49.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:13:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:50.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:13:50.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.0 on smithi064 2024-08-17T17:13:50.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.0 on smithi064 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:13:50.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.0 on smithi064 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:13:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.1 on smithi064 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.1 on smithi064 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:13:51.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:51.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:13:51.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.1 on smithi064 2024-08-17T17:13:51.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:51.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:13:51.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[104516]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[104516]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:13:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.2 on smithi064 2024-08-17T17:13:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[109246]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[109246]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:13:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:52 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.2 on smithi064 2024-08-17T17:13:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:52 smithi175 ceph-mon[101434]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:52 smithi175 ceph-mon[101434]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:13:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:52 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.2 on smithi064 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.3 on smithi064 2024-08-17T17:13:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:53 smithi175 ceph-mon[101434]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.3 on smithi064 2024-08-17T17:13:53.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[104516]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.3 on smithi064 2024-08-17T17:13:53.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-mon[109246]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:13:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:13:53] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.51.0" 2024-08-17T17:13:55.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: Reconfiguring mds.foo.smithi064.qahkwa (monmap changed)... 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[104516]: Reconfiguring daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: Reconfiguring mds.foo.smithi064.qahkwa (monmap changed)... 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:55.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:55 smithi064 ceph-mon[109246]: Reconfiguring daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: Reconfiguring mds.foo.smithi064.qahkwa (monmap changed)... 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.qahkwa", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:55 smithi175 ceph-mon[101434]: Reconfiguring daemon mds.foo.smithi064.qahkwa on smithi064 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:56.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:13:56.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:13:57.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: Reconfiguring daemon mon.b on smithi175 2024-08-17T17:13:57.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:57.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:13:57.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:13:57.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:58.336 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:58 smithi175 ceph-mon[101434]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:58.337 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:58 smithi175 ceph-mon[101434]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:13:58.337 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:58 smithi175 ceph-mon[101434]: Reconfiguring daemon mgr.x on smithi175 2024-08-17T17:13:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[104516]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:58.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[104516]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:13:58.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[104516]: Reconfiguring daemon mgr.x on smithi175 2024-08-17T17:13:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[109246]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[109246]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:13:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:58 smithi064 ceph-mon[109246]: Reconfiguring daemon mgr.x on smithi175 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:13:59 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.4 on smithi175 2024-08-17T17:13:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:13:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.4 on smithi175 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:13:59 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.4 on smithi175 2024-08-17T17:14:00.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:00 smithi175 ceph-mon[101434]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:14:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[104516]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[109246]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:14:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:01.593 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:14:01.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.5 on smithi175 2024-08-17T17:14:01.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:14:01.594 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:14:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.5 on smithi175 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.5 on smithi175 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:14:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:02.287 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:14:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:02 smithi175 ceph-mon[101434]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:02 smithi175 ceph-mon[101434]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:14:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:02 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.6 on smithi175 2024-08-17T17:14:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[104516]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[104516]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:14:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.6 on smithi175 2024-08-17T17:14:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[109246]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[109246]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:14:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:02 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.6 on smithi175 2024-08-17T17:14:02.870 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:14:02.870 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (3m) 67s ago 9m 20.2M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:14:02.870 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (3m) 22s ago 9m 77.2M - c8b91775d855 92fb2e1fda05 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (8m) 67s ago 8m 18.5M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (8m) 22s ago 8m 18.5M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (4m) 22s ago 13m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (5m) 67s ago 15m 508M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (2m) 67s ago 15m 40.7M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (28s) 22s ago 13m 25.7M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9bdd73bb1f26 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (2m) 67s ago 13m 32.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (4m) 67s ago 9m 10.9M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (4m) 22s ago 9m 10.7M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (13m) 67s ago 13m 58.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (12m) 67s ago 12m 53.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (12m) 67s ago 12m 47.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:14:02.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (11m) 67s ago 11m 52.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:14:02.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (11m) 22s ago 11m 49.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:14:02.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (11m) 22s ago 11m 54.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:14:02.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (10m) 22s ago 10m 58.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:14:02.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (10m) 22s ago 10m 55.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:14:02.872 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (3m) 22s ago 9m 45.9M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:14:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 10, 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 5 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:14:03.226 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [ 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mon" 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:14:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "3/3 daemons upgraded", 2024-08-17T17:14:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:14:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:14:03.528 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:14:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: Reconfiguring daemon osd.7 on smithi175 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/87137' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: Reconfiguring daemon osd.7 on smithi175 2024-08-17T17:14:03.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/87137' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: Reconfiguring daemon osd.7 on smithi175 2024-08-17T17:14:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/87137' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:04.387 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:03] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.51.0" 2024-08-17T17:14:04.503 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.503 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:04.503 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.503 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.504 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.504 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:14:04.504 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:04.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:04.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:04.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi175.nhcgie", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:14:04.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: Reconfiguring mds.foo.smithi175.nhcgie (monmap changed)... 2024-08-17T17:14:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: Reconfiguring daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:14:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:14:05.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:14:05.663 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:14:05.664 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:14:05.665 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: Reconfiguring mds.foo.smithi175.nhcgie (monmap changed)... 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: Reconfiguring daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:05.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: Reconfiguring mds.foo.smithi175.nhcgie (monmap changed)... 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: Reconfiguring daemon mds.foo.smithi175.nhcgie on smithi175 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:14:05.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:14:05.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.696 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:14:05.696 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:14:05.696 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:14:05.696 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all grafana 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all loki 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all promtail 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Finalizing container_image settings 2024-08-17T17:14:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:06 smithi175 ceph-mon[101434]: Upgrade: Complete! 2024-08-17T17:14:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:14:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all grafana 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all loki 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all promtail 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Finalizing container_image settings 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[104516]: Upgrade: Complete! 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:14:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all grafana 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all loki 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all promtail 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Finalizing container_image settings 2024-08-17T17:14:06.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:06 smithi064 ceph-mon[109246]: Upgrade: Complete! 2024-08-17T17:14:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:08 smithi175 ceph-mon[101434]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:08.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:08 smithi064 ceph-mon[104516]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:08 smithi064 ceph-mon[109246]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:10.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:11.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:11.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:14:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:12 smithi175 ceph-mon[101434]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:12 smithi064 ceph-mon[104516]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:12 smithi064 ceph-mon[109246]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:14 smithi175 ceph-mon[101434]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:14 smithi064 ceph-mon[104516]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:14.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:13] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.51.0" 2024-08-17T17:14:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:14 smithi064 ceph-mon[109246]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:16 smithi175 ceph-mon[101434]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:16 smithi064 ceph-mon[104516]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:16 smithi064 ceph-mon[109246]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:18 smithi175 ceph-mon[101434]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:18 smithi064 ceph-mon[104516]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:18 smithi064 ceph-mon[109246]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:20 smithi175 ceph-mon[101434]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:20.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:20 smithi064 ceph-mon[104516]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:20.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:20 smithi064 ceph-mon[109246]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:22.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:22 smithi175 ceph-mon[101434]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:22 smithi064 ceph-mon[104516]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:22.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:22 smithi064 ceph-mon[109246]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:24.407 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:24 smithi175 ceph-mon[101434]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:24 smithi064 ceph-mon[104516]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:24.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:23] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.51.0" 2024-08-17T17:14:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:24 smithi064 ceph-mon[109246]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:25.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:25.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:26 smithi175 ceph-mon[101434]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:26 smithi064 ceph-mon[104516]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:26.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:26 smithi064 ceph-mon[109246]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:28 smithi175 ceph-mon[101434]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:28 smithi064 ceph-mon[104516]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:28 smithi064 ceph-mon[109246]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:30 smithi175 ceph-mon[101434]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:30 smithi064 ceph-mon[104516]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:30 smithi064 ceph-mon[109246]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:32 smithi175 ceph-mon[101434]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:32 smithi064 ceph-mon[104516]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:32 smithi064 ceph-mon[109246]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:34.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:34 smithi064 ceph-mon[104516]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:34.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:33] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.51.0" 2024-08-17T17:14:34.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:34 smithi064 ceph-mon[109246]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:34 smithi175 ceph-mon[101434]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:34.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:14:34.700 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:35 smithi175 ceph-mon[101434]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:35 smithi064 ceph-mon[104516]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:35 smithi064 ceph-mon[109246]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:14:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:36 smithi175 ceph-mon[101434]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:36 smithi064 ceph-mon[104516]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:36.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:36 smithi064 ceph-mon[109246]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:38 smithi175 ceph-mon[101434]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:38 smithi064 ceph-mon[104516]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:38.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:38 smithi064 ceph-mon[109246]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:40 smithi175 ceph-mon[101434]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:40 smithi064 ceph-mon[104516]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:40.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:40 smithi064 ceph-mon[109246]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:40.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:42 smithi175 ceph-mon[101434]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:42 smithi064 ceph-mon[104516]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:42 smithi064 ceph-mon[109246]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:44 smithi175 ceph-mon[101434]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:44.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:44 smithi064 ceph-mon[104516]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:44.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:44 smithi064 ceph-mon[109246]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:43] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-08-17T17:14:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:46 smithi175 ceph-mon[101434]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:46 smithi064 ceph-mon[104516]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:46 smithi064 ceph-mon[109246]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:48 smithi175 ceph-mon[101434]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:48 smithi064 ceph-mon[104516]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:48.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:48 smithi064 ceph-mon[109246]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:50 smithi175 ceph-mon[101434]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:50 smithi064 ceph-mon[104516]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:50.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:50 smithi064 ceph-mon[109246]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:52.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:52 smithi175 ceph-mon[101434]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:52.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:52 smithi064 ceph-mon[104516]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:52.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:52 smithi064 ceph-mon[109246]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:54 smithi175 ceph-mon[101434]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:54.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:54 smithi064 ceph-mon[104516]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:54.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:54 smithi064 ceph-mon[109246]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:54.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:14:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:14:53] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-08-17T17:14:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:56 smithi175 ceph-mon[101434]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:56 smithi064 ceph-mon[104516]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:56 smithi064 ceph-mon[109246]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:14:58 smithi175 ceph-mon[101434]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:14:58 smithi064 ceph-mon[104516]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:14:58.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:14:58 smithi064 ceph-mon[109246]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:00 smithi175 ceph-mon[101434]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:00.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:00 smithi064 ceph-mon[109246]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:00 smithi064 ceph-mon[104516]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:02 smithi175 ceph-mon[101434]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:02 smithi064 ceph-mon[104516]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:02.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:02 smithi064 ceph-mon[109246]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:04.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:04 smithi175 ceph-mon[101434]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:04 smithi064 ceph-mon[104516]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:04.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:04 smithi064 ceph-mon[109246]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:04.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:03] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.51.0" 2024-08-17T17:15:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:06 smithi175 ceph-mon[101434]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:06 smithi064 ceph-mon[104516]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:06 smithi064 ceph-mon[109246]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:08 smithi175 ceph-mon[101434]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:08 smithi064 ceph-mon[104516]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:08.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:08 smithi064 ceph-mon[109246]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:10.311 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:10 smithi175 ceph-mon[101434]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:10.311 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:10.311 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[104516]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[109246]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:12 smithi175 ceph-mon[101434]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:12.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:12 smithi064 ceph-mon[104516]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:12.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:12 smithi064 ceph-mon[109246]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:13 smithi175 ceph-mon[101434]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:13] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-08-17T17:15:14.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[104516]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:14.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[109246]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:14.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:14.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:14 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:14 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:14 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:15.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:15.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:15.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:15.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:15.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:15.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:14 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:16.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:15 smithi175 ceph-mon[101434]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:15 smithi064 ceph-mon[104516]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:16.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:15 smithi064 ceph-mon[109246]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:17 smithi175 ceph-mon[101434]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:17 smithi064 ceph-mon[104516]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:17 smithi064 ceph-mon[109246]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:19 smithi175 ceph-mon[101434]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:19 smithi064 ceph-mon[104516]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:19 smithi064 ceph-mon[109246]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:21 smithi175 ceph-mon[101434]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:22.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:21 smithi064 ceph-mon[104516]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:22.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:21 smithi064 ceph-mon[109246]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:23 smithi175 ceph-mon[101434]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:24.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:23 smithi064 ceph-mon[104516]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:24.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:23] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-08-17T17:15:24.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:23 smithi064 ceph-mon[109246]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:25 smithi175 ceph-mon[101434]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:25 smithi064 ceph-mon[104516]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:25 smithi064 ceph-mon[109246]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:27 smithi175 ceph-mon[101434]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:28.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:27 smithi064 ceph-mon[104516]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:28.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:27 smithi064 ceph-mon[109246]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:29 smithi175 ceph-mon[101434]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:29 smithi064 ceph-mon[104516]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:30.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:29 smithi064 ceph-mon[109246]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:31 smithi175 ceph-mon[101434]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:31 smithi064 ceph-mon[104516]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:32.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:31 smithi064 ceph-mon[109246]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:33 smithi175 ceph-mon[101434]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:33 smithi064 ceph-mon[104516]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:33] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-08-17T17:15:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:33 smithi064 ceph-mon[109246]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:36.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:35 smithi175 ceph-mon[101434]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:36.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:35 smithi064 ceph-mon[104516]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:36.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:35 smithi064 ceph-mon[109246]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:36.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for all mons to upgrade' 2024-08-17T17:15:36.982 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:38.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:37 smithi064 ceph-mon[104516]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:38.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:37 smithi064 ceph-mon[109246]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:37 smithi175 ceph-mon[101434]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:38.563 INFO:teuthology.orchestra.run.smithi064.stdout:wait for all mons to upgrade 2024-08-17T17:15:39.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:40.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:39 smithi064 ceph-mon[104516]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:40.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:39 smithi064 ceph-mon[109246]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:40.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:39 smithi175 ceph-mon[101434]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:41.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:41.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:41.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:42.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:41 smithi064 ceph-mon[104516]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:42.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:41 smithi064 ceph-mon[104516]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:42.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:41 smithi064 ceph-mon[109246]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:42.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:41 smithi064 ceph-mon[109246]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:42.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:41 smithi175 ceph-mon[101434]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:42.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:41 smithi175 ceph-mon[101434]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:43.041 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:43.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:15:43.862 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:44.209 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:43] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.51.0" 2024-08-17T17:15:44.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:44.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:44.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[104516]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:44.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:44.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:44.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:44 smithi064 ceph-mon[109246]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:44.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:44 smithi175 ceph-mon[101434]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:15:46.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:46 smithi064 ceph-mon[104516]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:46.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:46 smithi064 ceph-mon[109246]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:46 smithi175 ceph-mon[101434]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:47.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:15:47.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.637 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:47.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:15:47.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:15:47.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all mgr 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all mon 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all crash 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all rgw 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:15:47.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all mgr 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all mon 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all crash 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all rgw 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:15:47.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:47.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all mgr 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all mon 2024-08-17T17:15:47.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all crash 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all rgw 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.915 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:47.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.918 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.919 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.919 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.919 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:15:47.919 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:15:47.919 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:15:47.920 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:15:47.921 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.922 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:47.923 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:48.756 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all nfs 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all grafana 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all loki 2024-08-17T17:15:48.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all promtail 2024-08-17T17:15:48.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Finalizing container_image settings 2024-08-17T17:15:48.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: Upgrade: Complete! 2024-08-17T17:15:48.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:48.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[109246]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:48.758 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:15:48.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:15:48.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:15:48.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all nfs 2024-08-17T17:15:48.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:15:48.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all grafana 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all loki 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all promtail 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Finalizing container_image settings 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: Upgrade: Complete! 2024-08-17T17:15:48.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:48 smithi064 ceph-mon[104516]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all nfs 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all grafana 2024-08-17T17:15:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all loki 2024-08-17T17:15:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all promtail 2024-08-17T17:15:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Finalizing container_image settings 2024-08-17T17:15:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: Upgrade: Complete! 2024-08-17T17:15:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:48 smithi175 ceph-mon[101434]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:50.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:50 smithi064 ceph-mon[104516]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:50.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:50.781 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:50 smithi064 ceph-mon[109246]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:50.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:50.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:50 smithi175 ceph-mon[101434]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:15:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:52 smithi175 ceph-mon[101434]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:52 smithi064 ceph-mon[104516]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:52 smithi064 ceph-mon[109246]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:54.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:15:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:15:53] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.51.0" 2024-08-17T17:15:54.713 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:54 smithi175 ceph-mon[101434]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:54 smithi064 ceph-mon[104516]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:54 smithi064 ceph-mon[109246]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:56 smithi175 ceph-mon[101434]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:56.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:56 smithi064 ceph-mon[104516]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:56.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:56 smithi064 ceph-mon[109246]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:15:58 smithi175 ceph-mon[101434]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:15:58 smithi064 ceph-mon[104516]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:15:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:15:58 smithi064 ceph-mon[109246]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:00 smithi175 ceph-mon[101434]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:00 smithi064 ceph-mon[104516]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:00 smithi064 ceph-mon[109246]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:02 smithi175 ceph-mon[101434]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:02 smithi064 ceph-mon[104516]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:02 smithi064 ceph-mon[109246]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:04.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:03] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.51.0" 2024-08-17T17:16:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:04 smithi175 ceph-mon[101434]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:04 smithi064 ceph-mon[104516]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:04.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:04 smithi064 ceph-mon[109246]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:06 smithi175 ceph-mon[101434]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:06 smithi064 ceph-mon[104516]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:06.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:06 smithi064 ceph-mon[109246]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:07 smithi175 ceph-mon[101434]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:07.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:07 smithi064 ceph-mon[104516]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:07.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:07 smithi064 ceph-mon[109246]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:09 smithi175 ceph-mon[101434]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:10.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:09 smithi064 ceph-mon[104516]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:10.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:09 smithi064 ceph-mon[109246]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:11.021 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:11.021 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:11.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:12.011 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:11 smithi064 ceph-mon[104516]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:12.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:11 smithi064 ceph-mon[109246]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:12.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:11 smithi175 ceph-mon[101434]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:13 smithi175 ceph-mon[101434]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:14.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:13 smithi064 ceph-mon[104516]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:14.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:13] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-08-17T17:16:14.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:13 smithi064 ceph-mon[109246]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:16.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:15 smithi175 ceph-mon[101434]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:15 smithi064 ceph-mon[104516]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:16.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:15 smithi064 ceph-mon[109246]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:17 smithi175 ceph-mon[101434]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:17 smithi064 ceph-mon[104516]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:17 smithi064 ceph-mon[109246]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:19 smithi175 ceph-mon[101434]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:19 smithi064 ceph-mon[104516]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:19 smithi064 ceph-mon[109246]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:21 smithi064 ceph-mon[104516]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:21 smithi064 ceph-mon[109246]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:21 smithi175 ceph-mon[101434]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:23 smithi175 ceph-mon[101434]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:24.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:23 smithi064 ceph-mon[104516]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:24.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:23] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-08-17T17:16:24.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:23 smithi064 ceph-mon[109246]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:25 smithi175 ceph-mon[101434]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:25 smithi064 ceph-mon[104516]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:25 smithi064 ceph-mon[109246]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:27 smithi175 ceph-mon[101434]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:28.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:27 smithi064 ceph-mon[109246]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:28.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:27 smithi064 ceph-mon[104516]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:29 smithi175 ceph-mon[101434]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:30.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:29 smithi064 ceph-mon[109246]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:29 smithi064 ceph-mon[104516]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:31 smithi175 ceph-mon[101434]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:32.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:31 smithi064 ceph-mon[109246]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:31 smithi064 ceph-mon[104516]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:33 smithi175 ceph-mon[101434]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:33 smithi064 ceph-mon[104516]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:33] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.51.0" 2024-08-17T17:16:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:33 smithi064 ceph-mon[109246]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:35 smithi175 ceph-mon[101434]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:35 smithi064 ceph-mon[104516]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:36.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:35 smithi064 ceph-mon[109246]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:37 smithi175 ceph-mon[101434]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:37 smithi064 ceph-mon[104516]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:37 smithi064 ceph-mon[109246]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:40.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:39 smithi175 ceph-mon[101434]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:40.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:39 smithi064 ceph-mon[104516]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:40.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:39 smithi064 ceph-mon[109246]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:41.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:41.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:42.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:41 smithi175 ceph-mon[101434]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:42.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:41 smithi064 ceph-mon[104516]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:42.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:41 smithi064 ceph-mon[109246]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:44.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:43 smithi175 ceph-mon[101434]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:43 smithi064 ceph-mon[104516]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:44.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:43 smithi064 ceph-mon[109246]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:44.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:43] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.51.0" 2024-08-17T17:16:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:45 smithi175 ceph-mon[101434]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:45 smithi064 ceph-mon[104516]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:46.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:45 smithi064 ceph-mon[109246]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:48.048 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:47 smithi175 ceph-mon[101434]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:48.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:47 smithi064 ceph-mon[104516]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:47 smithi064 ceph-mon[109246]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:49.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for half of osds to upgrade' 2024-08-17T17:16:49.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:49 smithi175 ceph-mon[101434]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:50.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:49 smithi064 ceph-mon[104516]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:50.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:49 smithi064 ceph-mon[109246]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:50.988 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:52.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:52.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:52.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:52.778 INFO:teuthology.orchestra.run.smithi064.stdout:wait for half of osds to upgrade 2024-08-17T17:16:53.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-08-17T17:16:53.632 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:54.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:53 smithi064 ceph-mon[104516]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:54.019 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:16:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:16:53] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.51.0" 2024-08-17T17:16:54.019 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:53 smithi064 ceph-mon[109246]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:53 smithi175 ceph-mon[101434]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:55 smithi175 ceph-mon[101434]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:55 smithi175 ceph-mon[101434]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[104516]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[104516]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[109246]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:56.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:55 smithi064 ceph-mon[109246]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:57.247 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:57.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:16:58.170 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:58.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[104516]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:16:58 smithi064 ceph-mon[109246]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:16:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:16:58 smithi175 ceph-mon[101434]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:17:00.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:00 smithi064 ceph-mon[104516]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:00.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:00 smithi064 ceph-mon[109246]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:00.618 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:17:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:00 smithi175 ceph-mon[101434]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:01.240 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (6m) 10s ago 12m 19.1M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (6m) 10s ago 12m 77.3M - c8b91775d855 92fb2e1fda05 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (11m) 10s ago 11m 20.2M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (11m) 10s ago 11m 19.4M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (7m) 10s ago 16m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (8m) 10s ago 18m 515M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (5m) 10s ago 18m 50.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (3m) 10s ago 16m 40.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9bdd73bb1f26 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (5m) 10s ago 16m 43.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (7m) 10s ago 12m 10.5M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:17:01.241 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (7m) 10s ago 12m 10.9M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (16m) 10s ago 16m 58.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 47373c53019b 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (15m) 10s ago 15m 54.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (15m) 10s ago 15m 48.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (14m) 10s ago 14m 53.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (14m) 10s ago 14m 50.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (14m) 10s ago 14m 55.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (13m) 10s ago 13m 59.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (13m) 10s ago 13m 56.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:17:01.242 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (6m) 10s ago 12m 50.2M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:17:01.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:01.601 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:17:01.601 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 10, 2024-08-17T17:17:01.601 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 5 2024-08-17T17:17:01.601 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:17:01.601 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:17:01.903 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "0/8 daemons upgraded", 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:17:01.904 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:17:02.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all mgr 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all mon 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all crash 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: Upgrade: osd.0 is safe to restart 2024-08-17T17:17:02.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2844008936' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all mgr 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all mon 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all crash 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: Upgrade: osd.0 is safe to restart 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2844008936' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all mgr 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all mon 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all crash 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: Upgrade: osd.0 is safe to restart 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:02.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2844008936' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: Upgrade: Updating osd.0 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:03 smithi175 ceph-mon[101434]: Deploying daemon osd.0 on smithi064 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: Upgrade: Updating osd.0 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:03.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[104516]: Deploying daemon osd.0 on smithi064 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: Upgrade: Updating osd.0 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:03.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-mon[109246]: Deploying daemon osd.0 on smithi064 2024-08-17T17:17:03.986 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:03] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.51.0" 2024-08-17T17:17:04.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:04 smithi175 ceph-mon[101434]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:04.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:04 smithi064 ceph-mon[109246]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:04 smithi064 ceph-mon[104516]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:05.275 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:05 smithi064 systemd[1]: Stopping Ceph osd.0 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:17:05.687 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[44957]: 2024-08-17T17:17:05.274+0000 7f7d4f60a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:17:05.687 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[44957]: 2024-08-17T17:17:05.274+0000 7f7d4f60a640 -1 osd.0 90 *** Got signal Terminated *** 2024-08-17T17:17:05.687 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:05 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[44957]: 2024-08-17T17:17:05.274+0000 7f7d4f60a640 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:17:06.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:06 smithi175 ceph-mon[101434]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:06 smithi175 ceph-mon[101434]: osd.0 marked itself down and dead 2024-08-17T17:17:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:06 smithi064 ceph-mon[109246]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:06 smithi064 ceph-mon[109246]: osd.0 marked itself down and dead 2024-08-17T17:17:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:06 smithi064 ceph-mon[104516]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:06 smithi064 ceph-mon[104516]: osd.0 marked itself down and dead 2024-08-17T17:17:06.892 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:06 smithi064 podman[120154]: 2024-08-17 17:17:06.543218035 +0000 UTC m=+1.373549543 container died 47373c53019bbfbf41da847eb78f6bb27e5b9a131d9dace19acda1a80f224f3d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, RELEASE=reef-ced649f, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:17:07.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:07 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:07.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:07 smithi064 ceph-mon[104516]: osdmap e91: 8 total, 7 up, 8 in 2024-08-17T17:17:07.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:07 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:07.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:07 smithi064 ceph-mon[109246]: osdmap e91: 8 total, 7 up, 8 in 2024-08-17T17:17:07.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:06 smithi064 podman[120154]: 2024-08-17 17:17:06.990896878 +0000 UTC m=+1.821228384 container cleanup 47373c53019bbfbf41da847eb78f6bb27e5b9a131d9dace19acda1a80f224f3d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:17:07.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:06 smithi064 bash[120154]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:17:07.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:07 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:07.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:07 smithi175 ceph-mon[101434]: osdmap e91: 8 total, 7 up, 8 in 2024-08-17T17:17:07.624 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:07 smithi064 podman[120171]: 2024-08-17 17:17:07.200749648 +0000 UTC m=+0.646947326 container remove 47373c53019bbfbf41da847eb78f6bb27e5b9a131d9dace19acda1a80f224f3d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f) 2024-08-17T17:17:07.896 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:07 smithi064 podman[120260]: 2024-08-17 17:17:07.527014552 +0000 UTC m=+0.017856018 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:07.896 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:07 smithi064 podman[120260]: 2024-08-17 17:17:07.62683651 +0000 UTC m=+0.117677995 container create cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:17:08.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:08 smithi064 ceph-mon[109246]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:08.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:08 smithi064 ceph-mon[109246]: osdmap e92: 8 total, 7 up, 8 in 2024-08-17T17:17:08.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:08 smithi064 ceph-mon[104516]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:08.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:08 smithi064 ceph-mon[104516]: osdmap e92: 8 total, 7 up, 8 in 2024-08-17T17:17:08.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:07 smithi064 podman[120260]: 2024-08-17 17:17:07.89599606 +0000 UTC m=+0.386837511 container init cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:17:08.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:07 smithi064 podman[120260]: 2024-08-17 17:17:07.899594579 +0000 UTC m=+0.390436032 container start cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:17:08.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 podman[120260]: 2024-08-17 17:17:08.012520942 +0000 UTC m=+0.503362397 container attach cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-08-17T17:17:08.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 podman[120260]: 2024-08-17 17:17:08.0946023 +0000 UTC m=+0.585443750 container died cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:17:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:08 smithi175 ceph-mon[101434]: pgmap v175: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:08 smithi175 ceph-mon[101434]: osdmap e92: 8 total, 7 up, 8 in 2024-08-17T17:17:08.937 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 podman[120283]: 2024-08-17 17:17:08.67427085 +0000 UTC m=+0.573668339 container remove cd86db2c3345b127a65a7a738e9a892a6de6d6f78b9c10e91d744841a99ffe2f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-08-17T17:17:08.937 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-17T17:17:08.937 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 systemd[1]: Stopped Ceph osd.0 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:17:08.938 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.0.service: Consumed 7.456s CPU time. 2024-08-17T17:17:08.938 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:08 smithi064 systemd[1]: Starting Ceph osd.0 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:17:09.420 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.117332977 +0000 UTC m=+0.100918612 container create a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:17:09.420 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.034758876 +0000 UTC m=+0.018344509 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:09.420 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.345408613 +0000 UTC m=+0.328994253 container init a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T17:17:09.420 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.349096809 +0000 UTC m=+0.332682447 container start a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:17:09.420 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.419851234 +0000 UTC m=+0.403436879 container attach a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:09.689 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:17:09.689 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:17:09.689 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:17:09.689 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[104516]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[109246]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:10.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:10.188 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate[120397]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 bash[120382]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:17:10.189 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:09 smithi064 podman[120382]: 2024-08-17 17:17:09.918075903 +0000 UTC m=+0.901661547 container died a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:17:10.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:10 smithi175 ceph-mon[101434]: pgmap v177: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:10.863 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 podman[120519]: 2024-08-17 17:17:10.515504433 +0000 UTC m=+0.586199110 container remove a9fd5f09229f05b60da065e7822f690bba6af9af594416748a5105d03fc3c37c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:17:10.863 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 podman[120580]: 2024-08-17 17:17:10.637178639 +0000 UTC m=+0.018066594 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:10.863 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 podman[120580]: 2024-08-17 17:17:10.758204525 +0000 UTC m=+0.139092466 container create 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T17:17:11.130 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 podman[120580]: 2024-08-17 17:17:10.960395912 +0000 UTC m=+0.341283866 container init 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:17:11.130 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 podman[120580]: 2024-08-17 17:17:10.964052202 +0000 UTC m=+0.344940143 container start 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe) 2024-08-17T17:17:11.130 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:10 smithi064 ceph-osd[120600]: -- 172.21.15.64:0/3799106139 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556c4ed39a40 con 0x556c4fb06000 2024-08-17T17:17:11.130 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:11 smithi064 bash[120580]: 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd 2024-08-17T17:17:11.130 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:11 smithi064 systemd[1]: Started Ceph osd.0 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:17:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:11 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:12.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:12 smithi175 ceph-mon[101434]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:12 smithi064 ceph-mon[104516]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:12.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:12 smithi064 ceph-mon[109246]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:13.187 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:17:12.879+0000 7f39b80cb640 -1 Falling back to public interface 2024-08-17T17:17:14.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:13] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.51.0" 2024-08-17T17:17:14.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:14 smithi175 ceph-mon[101434]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:14 smithi064 ceph-mon[104516]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:14.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:14 smithi064 ceph-mon[109246]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:15.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:16 smithi175 ceph-mon[101434]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:16.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:16 smithi064 ceph-mon[104516]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:16.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:16 smithi064 ceph-mon[109246]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:16.671 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:17:16.411+0000 7f39b80cb640 -1 osd.0 90 log_to_monitors true 2024-08-17T17:17:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:17 smithi175 ceph-mon[101434]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:17:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:17 smithi064 ceph-mon[104516]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:17:17.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:17 smithi064 ceph-mon[109246]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:17:17.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:17:17 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:17:17.395+0000 7f39afe75640 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:17:18.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:17:18.168+0000 7f06693d0640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: osdmap e93: 8 total, 7 up, 8 in 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: osdmap e93: 8 total, 7 up, 8 in 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: pgmap v181: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: osdmap e93: 8 total, 7 up, 8 in 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:18.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150] boot 2024-08-17T17:17:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:17:19.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:17:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150] boot 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: osd.0 [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150] boot 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:17:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:17:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:20 smithi175 ceph-mon[101434]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:20 smithi175 ceph-mon[101434]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:17:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:20 smithi064 ceph-mon[104516]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:20 smithi064 ceph-mon[104516]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:17:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:20 smithi064 ceph-mon[109246]: pgmap v184: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-17T17:17:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:20 smithi064 ceph-mon[109246]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:17:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:21 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:21 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:21 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:22 smithi064 ceph-mon[104516]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:22 smithi064 ceph-mon[109246]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:22 smithi175 ceph-mon[101434]: pgmap v186: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:24 smithi175 ceph-mon[101434]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:24 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:17:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:24 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:17:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[104516]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:17:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:17:24.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:23] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.51.0" 2024-08-17T17:17:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[109246]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:17:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:24 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:17:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:26 smithi175 ceph-mon[101434]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[104516]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[109246]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:28 smithi175 ceph-mon[101434]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:28 smithi064 ceph-mon[104516]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:28 smithi064 ceph-mon[109246]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:30 smithi175 ceph-mon[101434]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:30 smithi064 ceph-mon[104516]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:30 smithi064 ceph-mon[109246]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:31.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:31.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:32.229 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:17:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:32 smithi175 ceph-mon[101434]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:32 smithi064 ceph-mon[104516]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:32.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:32 smithi064 ceph-mon[109246]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (7m) 18s ago 13m 19.1M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (6m) 42s ago 12m 77.3M - c8b91775d855 92fb2e1fda05 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (11m) 18s ago 11m 20.4M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (11m) 42s ago 11m 19.4M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (8m) 42s ago 16m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (8m) 18s ago 18m 515M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (6m) 18s ago 18m 50.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:17:32.815 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (3m) 42s ago 17m 40.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9bdd73bb1f26 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (5m) 18s ago 17m 42.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (7m) 18s ago 13m 10.7M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (7m) 42s ago 13m 10.9M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (22s) 18s ago 16m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 63f96c51dd26 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (16m) 18s ago 16m 54.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 fec40e885736 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (15m) 18s ago 15m 48.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (15m) 18s ago 15m 54.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (15m) 42s ago 15m 50.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (14m) 42s ago 14m 55.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (14m) 42s ago 14m 59.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (13m) 42s ago 13m 56.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:17:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (7m) 42s ago 13m 50.2M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:17:33.166 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 7, 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:17:33.167 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 9, 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 6 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:17:33.168 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:17:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:33 smithi175 ceph-mon[101434]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:33 smithi064 ceph-mon[104516]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:33.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:33 smithi064 ceph-mon[109246]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:33.486 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "1/8 daemons upgraded", 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:17:33.487 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:17:34.091 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:33] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.51.0" 2024-08-17T17:17:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3953754273' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:34 smithi175 ceph-mon[101434]: Upgrade: osd.1 is safe to restart 2024-08-17T17:17:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3953754273' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[104516]: Upgrade: osd.1 is safe to restart 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3953754273' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:17:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:34 smithi064 ceph-mon[109246]: Upgrade: osd.1 is safe to restart 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[104516]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[109246]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:17:35.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:35.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:35 smithi175 ceph-mon[101434]: from='client.44258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:17:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[104516]: Upgrade: Updating osd.1 2024-08-17T17:17:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[104516]: Deploying daemon osd.1 on smithi064 2024-08-17T17:17:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[104516]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[109246]: Upgrade: Updating osd.1 2024-08-17T17:17:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[109246]: Deploying daemon osd.1 on smithi064 2024-08-17T17:17:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:36 smithi064 ceph-mon[109246]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:36 smithi175 ceph-mon[101434]: Upgrade: Updating osd.1 2024-08-17T17:17:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:36 smithi175 ceph-mon[101434]: Deploying daemon osd.1 on smithi064 2024-08-17T17:17:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:36 smithi175 ceph-mon[101434]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:37.721 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:37 smithi064 systemd[1]: Stopping Ceph osd.1 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:17:38.099 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[50614]: 2024-08-17T17:17:37.729+0000 7f835cdb7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:17:38.099 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[50614]: 2024-08-17T17:17:37.730+0000 7f835cdb7640 -1 osd.1 95 *** Got signal Terminated *** 2024-08-17T17:17:38.099 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:37 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[50614]: 2024-08-17T17:17:37.730+0000 7f835cdb7640 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:17:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:38 smithi175 ceph-mon[101434]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:38 smithi175 ceph-mon[101434]: osd.1 marked itself down and dead 2024-08-17T17:17:38.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:38 smithi064 ceph-mon[104516]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:38.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:38 smithi064 ceph-mon[104516]: osd.1 marked itself down and dead 2024-08-17T17:17:38.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:38 smithi064 ceph-mon[109246]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:38.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:38 smithi064 ceph-mon[109246]: osd.1 marked itself down and dead 2024-08-17T17:17:38.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:38 smithi064 podman[125174]: 2024-08-17 17:17:38.420975168 +0000 UTC m=+0.811613037 container died fec40e885736ba4b7eda0da76a6f26482313658ae5d3fd553d3ba1677e98b542 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-ced649f) 2024-08-17T17:17:39.015 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:38 smithi064 podman[125174]: 2024-08-17 17:17:38.838243384 +0000 UTC m=+1.228881249 container cleanup fec40e885736ba4b7eda0da76a6f26482313658ae5d3fd553d3ba1677e98b542 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:17:39.015 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:38 smithi064 bash[125174]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:17:39.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:39 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:39 smithi175 ceph-mon[101434]: osdmap e96: 8 total, 7 up, 8 in 2024-08-17T17:17:39.437 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125199]: 2024-08-17 17:17:39.014536233 +0000 UTC m=+0.591235622 container remove fec40e885736ba4b7eda0da76a6f26482313658ae5d3fd553d3ba1677e98b542 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T17:17:39.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:39 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:39.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:39 smithi064 ceph-mon[104516]: osdmap e96: 8 total, 7 up, 8 in 2024-08-17T17:17:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:39 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:17:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:39 smithi064 ceph-mon[109246]: osdmap e96: 8 total, 7 up, 8 in 2024-08-17T17:17:39.727 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.368656878 +0000 UTC m=+0.017725950 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:39.727 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.485083031 +0000 UTC m=+0.134152095 container create 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:17:40.116 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.726676057 +0000 UTC m=+0.375745120 container init 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:17:40.117 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.73097324 +0000 UTC m=+0.380042309 container start 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:17:40.117 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.810089262 +0000 UTC m=+0.459158328 container attach 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:17:40.117 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:39 smithi064 podman[125291]: 2024-08-17 17:17:39.932068019 +0000 UTC m=+0.581137082 container died 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-17T17:17:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:40 smithi175 ceph-mon[101434]: pgmap v196: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:40 smithi175 ceph-mon[101434]: osdmap e97: 8 total, 7 up, 8 in 2024-08-17T17:17:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[104516]: pgmap v196: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[104516]: osdmap e97: 8 total, 7 up, 8 in 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[109246]: pgmap v196: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[109246]: osdmap e97: 8 total, 7 up, 8 in 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:40.937 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 podman[125314]: 2024-08-17 17:17:40.513584193 +0000 UTC m=+0.575937790 container remove 3cbd82218f20a494dc24c3a34b865b9c5882b53ac2b8491aacff6165df339b67 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T17:17:40.938 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.1.service: Deactivated successfully. 2024-08-17T17:17:40.938 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 systemd[1]: Stopped Ceph osd.1 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:17:40.938 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.1.service: Consumed 7.812s CPU time. 2024-08-17T17:17:40.938 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 systemd[1]: Starting Ceph osd.1 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:17:41.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:41.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:41.335 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 podman[125414]: 2024-08-17 17:17:40.883687502 +0000 UTC m=+0.017570407 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:41.335 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:40 smithi064 podman[125414]: 2024-08-17 17:17:40.989595504 +0000 UTC m=+0.123478402 container create c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:17:41.335 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 podman[125414]: 2024-08-17 17:17:41.217638124 +0000 UTC m=+0.351521018 container init c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:17:41.335 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 podman[125414]: 2024-08-17 17:17:41.221338345 +0000 UTC m=+0.355221239 container start c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-08-17T17:17:41.335 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 podman[125414]: 2024-08-17 17:17:41.333925114 +0000 UTC m=+0.467808008 container attach c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:17:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:41 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:41.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:17:41.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:17:41.689 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate[125429]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 bash[125414]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:17:42.079 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:41 smithi064 podman[125414]: 2024-08-17 17:17:41.781580424 +0000 UTC m=+0.915463324 container died c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T17:17:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:42 smithi064 ceph-mon[104516]: pgmap v198: 73 pgs: 25 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:42 smithi064 ceph-mon[109246]: pgmap v198: 73 pgs: 25 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:42 smithi175 ceph-mon[101434]: pgmap v198: 73 pgs: 25 active+undersized, 1 stale+active+clean, 3 active+undersized+degraded, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:17:42.803 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 podman[125550]: 2024-08-17 17:17:42.456280124 +0000 UTC m=+0.668444902 container remove c2eef833debc86bebc8d59c9212b2c505d9c405e204f80161bb7f5deca7bd36d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-08-17T17:17:42.803 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 podman[125613]: 2024-08-17 17:17:42.58466667 +0000 UTC m=+0.018312500 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:17:42.803 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 podman[125613]: 2024-08-17 17:17:42.705285934 +0000 UTC m=+0.138931763 container create b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-08-17T17:17:43.187 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 podman[125613]: 2024-08-17 17:17:42.899559174 +0000 UTC m=+0.333205012 container init b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-08-17T17:17:43.187 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 podman[125613]: 2024-08-17 17:17:42.90314318 +0000 UTC m=+0.336789011 container start b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T17:17:43.187 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 ceph-osd[125632]: -- 172.21.15.64:0/3068960973 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55def7df3a40 con 0x55def8bc0400 2024-08-17T17:17:43.188 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 bash[125613]: b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 2024-08-17T17:17:43.188 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:42 smithi064 systemd[1]: Started Ceph osd.1 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:17:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:44 smithi175 ceph-mon[101434]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:44 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:43] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.51.0" 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[104516]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[109246]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:44.937 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:44 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:17:44.529+0000 7f11024f4640 -1 Falling back to public interface 2024-08-17T17:17:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:46 smithi175 ceph-mon[101434]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:46 smithi064 ceph-mon[104516]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:46 smithi064 ceph-mon[109246]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:48.437 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:17:48.062+0000 7f11024f4640 -1 osd.1 95 log_to_monitors true 2024-08-17T17:17:48.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[104516]: pgmap v201: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:48.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:48.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[104516]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:17:48.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:17:48.470+0000 7f10fa29e640 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:17:48.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[109246]: pgmap v201: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:48.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:48.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:48 smithi064 ceph-mon[109246]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:17:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:48 smithi175 ceph-mon[101434]: pgmap v201: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:48 smithi175 ceph-mon[101434]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:17:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:49 smithi175 ceph-mon[101434]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:49 smithi175 ceph-mon[101434]: osdmap e98: 8 total, 7 up, 8 in 2024-08-17T17:17:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:49 smithi175 ceph-mon[101434]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:49.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[104516]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:49.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[104516]: osdmap e98: 8 total, 7 up, 8 in 2024-08-17T17:17:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[104516]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:49.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[109246]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:49.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[109246]: osdmap e98: 8 total, 7 up, 8 in 2024-08-17T17:17:49.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:49 smithi064 ceph-mon[109246]: from='osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:17:50.583 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:17:50.449+0000 7f06693d0640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:50.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: pgmap v203: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:50.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757] boot 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: pgmap v203: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757] boot 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:50.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:50.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: pgmap v203: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: osd.1 [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757] boot 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:50.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:17:50.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:17:51.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:51 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:17:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:51 smithi175 ceph-mon[101434]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:51 smithi175 ceph-mon[101434]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[104516]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[104516]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[109246]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:17:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:51 smithi064 ceph-mon[109246]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:17:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:52 smithi175 ceph-mon[101434]: pgmap v206: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:17:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:52 smithi064 ceph-mon[104516]: pgmap v206: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:17:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:52 smithi064 ceph-mon[109246]: pgmap v206: 73 pgs: 9 active+undersized, 2 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:17:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:53.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:17:54.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:17:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:17:53] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.51.0" 2024-08-17T17:17:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:54 smithi175 ceph-mon[101434]: pgmap v207: 73 pgs: 7 active+undersized, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-17T17:17:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:54 smithi064 ceph-mon[104516]: pgmap v207: 73 pgs: 7 active+undersized, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-17T17:17:54.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:54 smithi064 ceph-mon[109246]: pgmap v207: 73 pgs: 7 active+undersized, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-17T17:17:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:56 smithi175 ceph-mon[101434]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:56 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-17T17:17:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:56 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:17:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[104516]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[109246]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-17T17:17:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:56 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:17:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:17:58 smithi175 ceph-mon[101434]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:17:58 smithi064 ceph-mon[104516]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:17:58.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:17:58 smithi064 ceph-mon[109246]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:00 smithi175 ceph-mon[101434]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:00 smithi064 ceph-mon[104516]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:00.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:00 smithi064 ceph-mon[109246]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:01.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:01.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:01.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:02.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:02 smithi175 ceph-mon[101434]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:02 smithi064 ceph-mon[104516]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:02.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:02 smithi064 ceph-mon[109246]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:03.813 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:18:04.095 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:03] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-08-17T17:18:04.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:04 smithi064 ceph-mon[104516]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:04.395 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:04 smithi064 ceph-mon[109246]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:04.395 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (7m) 17s ago 13m 19.2M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (7m) 73s ago 13m 77.3M - c8b91775d855 92fb2e1fda05 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (12m) 17s ago 12m 20.6M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (12m) 73s ago 12m 19.4M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (8m) 73s ago 17m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (9m) 17s ago 19m 516M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (6m) 17s ago 19m 55.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (4m) 73s ago 17m 40.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9bdd73bb1f26 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (6m) 17s ago 17m 45.4M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (8m) 17s ago 13m 10.5M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (8m) 73s ago 13m 10.9M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:18:04.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (53s) 17s ago 17m 40.3M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 63f96c51dd26 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (21s) 17s ago 16m 14.5M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb b136792f5b7c 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (16m) 17s ago 16m 49.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 6cd54b92b2b1 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (15m) 17s ago 15m 55.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (15m) 73s ago 15m 50.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (15m) 73s ago 15m 55.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (14m) 73s ago 14m 59.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (14m) 73s ago 14m 56.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:18:04.397 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (7m) 73s ago 14m 50.2M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:18:04.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:04 smithi175 ceph-mon[101434]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:04.747 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 6, 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:18:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8, 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 7 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:18:04.749 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "2/8 daemons upgraded", 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:18:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:18:05.045 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:18:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:05 smithi175 ceph-mon[101434]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:05 smithi175 ceph-mon[101434]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:05 smithi175 ceph-mon[101434]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2685911514' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[104516]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[104516]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[104516]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2685911514' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[109246]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[109246]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[109246]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2685911514' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:06.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[104516]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:06.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[104516]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:06.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:18:06.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:18:06.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[104516]: Upgrade: osd.2 is safe to restart 2024-08-17T17:18:06.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[109246]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:06.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[109246]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:06.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:18:06.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:18:06.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:06 smithi064 ceph-mon[109246]: Upgrade: osd.2 is safe to restart 2024-08-17T17:18:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:06 smithi175 ceph-mon[101434]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:06 smithi175 ceph-mon[101434]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:06 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:18:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:06 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:18:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:06 smithi175 ceph-mon[101434]: Upgrade: osd.2 is safe to restart 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: Upgrade: Updating osd.2 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[104516]: Deploying daemon osd.2 on smithi064 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:08.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: Upgrade: Updating osd.2 2024-08-17T17:18:08.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:08.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:18:08.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:08.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:07 smithi064 ceph-mon[109246]: Deploying daemon osd.2 on smithi064 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: Upgrade: Updating osd.2 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:07 smithi175 ceph-mon[101434]: Deploying daemon osd.2 on smithi064 2024-08-17T17:18:09.747 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:09 smithi064 systemd[1]: Stopping Ceph osd.2 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:18:10.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:09 smithi175 ceph-mon[101434]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:10.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:09 smithi064 ceph-mon[104516]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:10.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:09 smithi064 ceph-mon[109246]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:10.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[56341]: 2024-08-17T17:18:09.802+0000 7f1d29931640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:18:10.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[56341]: 2024-08-17T17:18:09.802+0000 7f1d29931640 -1 osd.2 100 *** Got signal Terminated *** 2024-08-17T17:18:10.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:09 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[56341]: 2024-08-17T17:18:09.802+0000 7f1d29931640 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:18:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:10 smithi175 ceph-mon[101434]: osd.2 marked itself down and dead 2024-08-17T17:18:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[104516]: osd.2 marked itself down and dead 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[109246]: osd.2 marked itself down and dead 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:11.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.429 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:11 smithi064 podman[130189]: 2024-08-17 17:18:11.167425085 +0000 UTC m=+1.468098793 container died 6cd54b92b2b1ede2e7a9b99fc88dded04189265a609bb4536feb3b221f63adbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-08-17T17:18:11.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:11 smithi064 podman[130189]: 2024-08-17 17:18:11.528512389 +0000 UTC m=+1.829186099 container cleanup 6cd54b92b2b1ede2e7a9b99fc88dded04189265a609bb4536feb3b221f63adbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:18:11.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:11 smithi064 bash[130189]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:18:12.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[104516]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:12.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:12.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[104516]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:18:12.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[109246]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:12.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:12.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:11 smithi064 ceph-mon[109246]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:18:12.148 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:11 smithi064 podman[130216]: 2024-08-17 17:18:11.805055389 +0000 UTC m=+0.634418666 container remove 6cd54b92b2b1ede2e7a9b99fc88dded04189265a609bb4536feb3b221f63adbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-08-17T17:18:12.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:11 smithi175 ceph-mon[101434]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:12.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:11 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:12.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:11 smithi175 ceph-mon[101434]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:18:12.400 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.147301488 +0000 UTC m=+0.108511728 container create dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:18:12.400 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.056519043 +0000 UTC m=+0.017729300 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:12.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.4003394 +0000 UTC m=+0.361549641 container init dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:18:12.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.40389142 +0000 UTC m=+0.365101667 container start dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:18:12.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.500350394 +0000 UTC m=+0.461560638 container attach dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-08-17T17:18:12.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:12 smithi064 podman[130306]: 2024-08-17 17:18:12.598813089 +0000 UTC m=+0.560023341 container died dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:18:13.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:12 smithi064 ceph-mon[104516]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:18:13.129 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:12 smithi064 ceph-mon[109246]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:18:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:12 smithi175 ceph-mon[101434]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:18:13.438 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130328]: 2024-08-17 17:18:13.128650018 +0000 UTC m=+0.524027987 container remove dcd233e9e533c5eb5ad5901a6b7053d3e0155608a529c7bbe160a25adef9ff15 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:18:13.438 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-17T17:18:13.438 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 systemd[1]: Stopped Ceph osd.2 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:18:13.438 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.2.service: Consumed 6.832s CPU time. 2024-08-17T17:18:13.438 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 systemd[1]: Starting Ceph osd.2 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:18:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-mon[104516]: pgmap v219: 73 pgs: 1 active+undersized, 3 stale+active+clean, 1 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:13.850 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130427]: 2024-08-17 17:18:13.48424063 +0000 UTC m=+0.018023883 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:13.850 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130427]: 2024-08-17 17:18:13.607398596 +0000 UTC m=+0.141181848 container create 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T17:18:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-mon[109246]: pgmap v219: 73 pgs: 1 active+undersized, 3 stale+active+clean, 1 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:14.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:13 smithi175 ceph-mon[101434]: pgmap v219: 73 pgs: 1 active+undersized, 3 stale+active+clean, 1 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:14.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:13 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:14.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:13] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.51.0" 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130427]: 2024-08-17 17:18:13.849259304 +0000 UTC m=+0.383042571 container init 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=) 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130427]: 2024-08-17 17:18:13.85315922 +0000 UTC m=+0.386942473 container start 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 podman[130427]: 2024-08-17 17:18:13.915786509 +0000 UTC m=+0.449569775 container attach 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:13 smithi064 bash[130427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:18:14.189 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:18:14.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate[130441]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 bash[130427]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:18:14.688 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 podman[130427]: 2024-08-17 17:18:14.410819832 +0000 UTC m=+0.944603087 container died 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-08-17T17:18:15.162 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:14 smithi064 podman[130553]: 2024-08-17 17:18:14.953069334 +0000 UTC m=+0.532221872 container remove 05e37a54da98809d394b0a3d1c64be4ad89ada8d1d256c4e27bafbf9b19f28fe (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:18:15.433 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 podman[130613]: 2024-08-17 17:18:15.162312718 +0000 UTC m=+0.107194685 container create 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-08-17T17:18:15.433 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 podman[130613]: 2024-08-17 17:18:15.073106795 +0000 UTC m=+0.017988735 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:15.433 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 podman[130613]: 2024-08-17 17:18:15.406481973 +0000 UTC m=+0.351363929 container init 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:18:15.433 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 podman[130613]: 2024-08-17 17:18:15.41008409 +0000 UTC m=+0.354966022 container start 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:18:15.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-osd[130632]: -- 172.21.15.64:0/2524268297 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e6f3491a40 con 0x55e6f346fc00 2024-08-17T17:18:15.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 bash[130613]: 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c 2024-08-17T17:18:15.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:15 smithi064 systemd[1]: Started Ceph osd.2 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:18:16.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[104516]: pgmap v220: 73 pgs: 5 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:16.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:16.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[109246]: pgmap v220: 73 pgs: 5 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:16.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:16.148 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:15 smithi175 ceph-mon[101434]: pgmap v220: 73 pgs: 5 active+undersized, 2 stale+active+clean, 1 active+undersized+degraded, 65 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:16.148 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.148 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.148 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:16.149 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:17.437 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:17 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:18:17.052+0000 7f2deff65640 -1 Falling back to public interface 2024-08-17T17:18:17.962 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:17 smithi175 ceph-mon[101434]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:17 smithi064 ceph-mon[104516]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:18.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:17 smithi064 ceph-mon[109246]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:19.093 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:19.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:19.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:20.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:20.884 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:18:20.607+0000 7f2deff65640 -1 osd.2 100 log_to_monitors true 2024-08-17T17:18:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:20 smithi175 ceph-mon[101434]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:18:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:20 smithi064 ceph-mon[104516]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:18:21.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:18:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:18:20.906+0000 7f2de7d0f640 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:18:21.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:20 smithi064 ceph-mon[109246]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:18:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:21 smithi175 ceph-mon[101434]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:21 smithi175 ceph-mon[101434]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:21 smithi175 ceph-mon[101434]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:18:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:21 smithi175 ceph-mon[101434]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:22.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[104516]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:22.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[104516]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:22.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[104516]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:18:22.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[104516]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:22.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[109246]: pgmap v223: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-17T17:18:22.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[109246]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:22.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[109246]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:18:22.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:21 smithi064 ceph-mon[109246]: from='osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:23.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:18:23.159+0000 7f06693d0640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[104516]: osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035] boot 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[104516]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[104516]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[109246]: osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035] boot 2024-08-17T17:18:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[109246]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:18:23.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:18:23.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:22 smithi064 ceph-mon[109246]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:18:23.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:22 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:23.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:22 smithi175 ceph-mon[101434]: osd.2 [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035] boot 2024-08-17T17:18:23.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:22 smithi175 ceph-mon[101434]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:18:23.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:18:23.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:22 smithi175 ceph-mon[101434]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:18:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:23] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.51.0" 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: pgmap v226: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:24.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:24.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[104516]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: pgmap v226: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:24.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:24.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:24.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:23 smithi064 ceph-mon[109246]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: pgmap v226: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:24.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:24.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:23 smithi175 ceph-mon[101434]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-17T17:18:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:26 smithi175 ceph-mon[101434]: pgmap v228: 73 pgs: 2 peering, 12 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[104516]: pgmap v228: 73 pgs: 2 peering, 12 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[109246]: pgmap v228: 73 pgs: 2 peering, 12 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:27.394 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:27 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:18:27.394 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:27 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:18:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:27 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:18:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:27 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:18:27.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:27 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-08-17T17:18:27.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:27 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:18:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:28 smithi175 ceph-mon[101434]: pgmap v229: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:28 smithi064 ceph-mon[104516]: pgmap v229: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:28 smithi064 ceph-mon[109246]: pgmap v229: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:30 smithi175 ceph-mon[101434]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:30 smithi064 ceph-mon[104516]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:30 smithi064 ceph-mon[109246]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:32 smithi175 ceph-mon[101434]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:32 smithi064 ceph-mon[104516]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:32.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:32 smithi064 ceph-mon[109246]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:34 smithi175 ceph-mon[101434]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:34 smithi064 ceph-mon[104516]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:33] "GET /metrics HTTP/1.1" 200 34005 "" "Prometheus/2.51.0" 2024-08-17T17:18:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:34 smithi064 ceph-mon[109246]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:35.370 INFO:teuthology.orchestra.run.smithi064.stdout:true 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (8m) 17s ago 14m 19.2M - 0.27.0 11f11916f8cd f2848ae5f362 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi175 *:3000 running (7m) 16s ago 13m 77.4M - c8b91775d855 92fb2e1fda05 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.qahkwa smithi064 running (12m) 17s ago 12m 20.9M - 18.2.4-719-gced649f8 c870753d67c2 e386640444c6 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi175.nhcgie smithi175 running (12m) 16s ago 12m 19.8M - 18.2.4-719-gced649f8 c870753d67c2 b3d20bef948b 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi175 *:8443,9283,8765 running (9m) 16s ago 18m 456M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 683d25124077 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (9m) 17s ago 19m 517M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb af5009d72be5 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (7m) 17s ago 19m 56.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 1eec4cd20a35 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi175 running (5m) 16s ago 18m 43.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9bdd73bb1f26 2024-08-17T17:18:35.952 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (6m) 17s ago 18m 46.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 00207a16fa93 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (8m) 17s ago 14m 11.1M - 1.7.0 72c9c2088986 7612fc9a72e1 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi175 *:9100 running (8m) 16s ago 14m 11.4M - 1.7.0 72c9c2088986 ee4f9a740541 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (85s) 17s ago 17m 40.4M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 63f96c51dd26 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (53s) 17s ago 17m 43.4M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb b136792f5b7c 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (20s) 17s ago 16m 11.7M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 8a5e7a009d59 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (16m) 17s ago 16m 55.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 bb7b8569ba21 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi175 running (16m) 16s ago 16m 52.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 8eb7c8820c01 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi175 running (15m) 16s ago 15m 55.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 656b5f8b3cd0 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi175 running (15m) 16s ago 15m 60.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 227f3eb1987a 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi175 running (14m) 16s ago 14m 57.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 7ccae91f6ae2 2024-08-17T17:18:35.953 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi175 *:9095 running (8m) 16s ago 14m 50.2M - 2.51.0 1d3b7f56885b 02b6969fb42d 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 5, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 7, 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 8 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:18:36.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[104516]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:36.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:36.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[104516]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:36.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[109246]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:36.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:36.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:36 smithi064 ceph-mon[109246]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:36 smithi175 ceph-mon[101434]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:36 smithi175 ceph-mon[101434]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:36.600 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "in_progress": true, 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "services_complete": [], 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "progress": "3/8 daemons upgraded", 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "is_paused": false 2024-08-17T17:18:36.601 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:18:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:37 smithi175 ceph-mon[101434]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:37 smithi175 ceph-mon[101434]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3635395533' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[104516]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[104516]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3635395533' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[109246]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[109246]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3635395533' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:38 smithi175 ceph-mon[101434]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:38 smithi175 ceph-mon[101434]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:38 smithi064 ceph-mon[104516]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:38 smithi064 ceph-mon[104516]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:38.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:38 smithi064 ceph-mon[109246]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:38 smithi064 ceph-mon[109246]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[104516]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[104516]: Upgrade: osd.3 is safe to restart 2024-08-17T17:18:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[109246]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:39 smithi064 ceph-mon[109246]: Upgrade: osd.3 is safe to restart 2024-08-17T17:18:39.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:39 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:18:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:39 smithi175 ceph-mon[101434]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:18:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:39 smithi175 ceph-mon[101434]: Upgrade: osd.3 is safe to restart 2024-08-17T17:18:40.321 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:40.322 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.323 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:41.339 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:41 smithi064 ceph-mon[104516]: Upgrade: Updating osd.3 2024-08-17T17:18:41.339 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:41 smithi064 ceph-mon[104516]: Deploying daemon osd.3 on smithi064 2024-08-17T17:18:41.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:41 smithi064 ceph-mon[109246]: Upgrade: Updating osd.3 2024-08-17T17:18:41.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:41 smithi064 ceph-mon[109246]: Deploying daemon osd.3 on smithi064 2024-08-17T17:18:41.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:41 smithi175 ceph-mon[101434]: Upgrade: Updating osd.3 2024-08-17T17:18:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:41 smithi175 ceph-mon[101434]: Deploying daemon osd.3 on smithi064 2024-08-17T17:18:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:42 smithi064 ceph-mon[104516]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:42.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:42 smithi064 ceph-mon[109246]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:42 smithi175 ceph-mon[101434]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:42.830 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:42 smithi064 systemd[1]: Stopping Ceph osd.3 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:18:43.122 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:42 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[62361]: 2024-08-17T17:18:42.828+0000 7f1e7d67a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:18:43.123 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:42 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[62361]: 2024-08-17T17:18:42.828+0000 7f1e7d67a640 -1 osd.3 105 *** Got signal Terminated *** 2024-08-17T17:18:43.123 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:42 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[62361]: 2024-08-17T17:18:42.828+0000 7f1e7d67a640 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:18:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:43 smithi175 ceph-mon[101434]: osd.3 marked itself down and dead 2024-08-17T17:18:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:43 smithi064 ceph-mon[104516]: osd.3 marked itself down and dead 2024-08-17T17:18:43.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:43 smithi064 ceph-mon[109246]: osd.3 marked itself down and dead 2024-08-17T17:18:43.905 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:43 smithi064 podman[135210]: 2024-08-17 17:18:43.597290262 +0000 UTC m=+0.870051059 container died bb7b8569ba21295856d64e0ab7d41f40093990f8dc16b0df490a5af16c929fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:18:44.189 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:43] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.51.0" 2024-08-17T17:18:44.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[109246]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:44.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:44.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[109246]: osdmap e106: 8 total, 7 up, 8 in 2024-08-17T17:18:44.190 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135210]: 2024-08-17 17:18:44.003746783 +0000 UTC m=+1.276507569 container cleanup bb7b8569ba21295856d64e0ab7d41f40093990f8dc16b0df490a5af16c929fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:18:44.190 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 bash[135210]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:18:44.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[104516]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:44.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:44.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:44 smithi064 ceph-mon[104516]: osdmap e106: 8 total, 7 up, 8 in 2024-08-17T17:18:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:44 smithi175 ceph-mon[101434]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:44 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:18:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:44 smithi175 ceph-mon[101434]: osdmap e106: 8 total, 7 up, 8 in 2024-08-17T17:18:44.681 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135227]: 2024-08-17 17:18:44.230213403 +0000 UTC m=+0.630418360 container remove bb7b8569ba21295856d64e0ab7d41f40093990f8dc16b0df490a5af16c929fd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:18:44.937 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135315]: 2024-08-17 17:18:44.681382671 +0000 UTC m=+0.107653707 container create 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2) 2024-08-17T17:18:44.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135315]: 2024-08-17 17:18:44.591544263 +0000 UTC m=+0.017815308 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:44.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135315]: 2024-08-17 17:18:44.875481344 +0000 UTC m=+0.301752399 container init 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:18:44.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135315]: 2024-08-17 17:18:44.879356752 +0000 UTC m=+0.305627788 container start 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-08-17T17:18:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:45 smithi175 ceph-mon[101434]: osdmap e107: 8 total, 7 up, 8 in 2024-08-17T17:18:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:45 smithi064 ceph-mon[104516]: osdmap e107: 8 total, 7 up, 8 in 2024-08-17T17:18:45.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:45 smithi064 ceph-mon[109246]: osdmap e107: 8 total, 7 up, 8 in 2024-08-17T17:18:45.437 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:44 smithi064 podman[135315]: 2024-08-17 17:18:44.975754139 +0000 UTC m=+0.402025184 container attach 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:18:45.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:45 smithi064 podman[135315]: 2024-08-17 17:18:45.075165857 +0000 UTC m=+0.501436893 container died 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:18:46.136 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:45 smithi064 podman[135339]: 2024-08-17 17:18:45.80500409 +0000 UTC m=+0.723645068 container remove 3a86a2ad463af0d7d1c520686b2c11a7ea3e7f55e5ee6f1107196cd35616ddef (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T17:18:46.136 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:45 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.3.service: Deactivated successfully. 2024-08-17T17:18:46.136 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:45 smithi064 systemd[1]: Stopped Ceph osd.3 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:18:46.136 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:45 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.3.service: Consumed 6.515s CPU time. 2024-08-17T17:18:46.136 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 systemd[1]: Starting Ceph osd.3 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:18:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:46 smithi175 ceph-mon[101434]: pgmap v240: 73 pgs: 5 peering, 9 stale+active+clean, 59 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:46 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:18:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-mon[104516]: pgmap v240: 73 pgs: 5 peering, 9 stale+active+clean, 59 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:18:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-mon[109246]: pgmap v240: 73 pgs: 5 peering, 9 stale+active+clean, 59 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:18:46.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 podman[135439]: 2024-08-17 17:18:46.157100811 +0000 UTC m=+0.018747216 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:46.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 podman[135439]: 2024-08-17 17:18:46.256481906 +0000 UTC m=+0.118128289 container create 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True) 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 podman[135439]: 2024-08-17 17:18:46.492703567 +0000 UTC m=+0.354349951 container init 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 podman[135439]: 2024-08-17 17:18:46.496161155 +0000 UTC m=+0.357807552 container start 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 podman[135439]: 2024-08-17 17:18:46.592473454 +0000 UTC m=+0.454119850 container attach 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:18:46.749 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:18:46.750 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:18:47.033 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:18:47.033 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:46 smithi064 bash[135439]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:18:47.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:47.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 bash[135439]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 bash[135439]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 bash[135439]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:18:47.355 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 bash[135439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:18:47.356 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate[135453]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:18:47.356 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 bash[135439]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:18:47.356 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 podman[135559]: 2024-08-17 17:18:47.088423131 +0000 UTC m=+0.018560741 container died 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T17:18:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:47 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:48.053 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 podman[135559]: 2024-08-17 17:18:47.755048915 +0000 UTC m=+0.685186560 container remove 7f1d8a7b20f3f8bd63f450d10d8981c20c351fa7e014c71752ffe1b99d130d4c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-17T17:18:48.053 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 podman[135620]: 2024-08-17 17:18:47.964365088 +0000 UTC m=+0.104623548 container create e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:18:48.053 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:47 smithi064 podman[135620]: 2024-08-17 17:18:47.877968317 +0000 UTC m=+0.018226805 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:18:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:48 smithi064 ceph-mon[104516]: pgmap v241: 73 pgs: 17 active+undersized, 5 peering, 2 stale+active+clean, 2 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:48.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:48 smithi064 ceph-mon[109246]: pgmap v241: 73 pgs: 17 active+undersized, 5 peering, 2 stale+active+clean, 2 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:48.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:48 smithi064 podman[135620]: 2024-08-17 17:18:48.14998676 +0000 UTC m=+0.290245244 container init e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:18:48.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:48 smithi064 podman[135620]: 2024-08-17 17:18:48.153281939 +0000 UTC m=+0.293540407 container start e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:18:48.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:48 smithi064 ceph-osd[135639]: -- 172.21.15.64:0/3826263640 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e19445da40 con 0x55e19443bc00 2024-08-17T17:18:48.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:48 smithi064 bash[135620]: e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 2024-08-17T17:18:48.438 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:48 smithi064 systemd[1]: Started Ceph osd.3 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:18:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:48 smithi175 ceph-mon[101434]: pgmap v241: 73 pgs: 17 active+undersized, 5 peering, 2 stale+active+clean, 2 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-17T17:18:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:50.100 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:18:49.824+0000 7f97afefb640 -1 Falling back to public interface 2024-08-17T17:18:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:50 smithi064 ceph-mon[104516]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:50 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-08-17T17:18:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:50 smithi064 ceph-mon[109246]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:50 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-08-17T17:18:50.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:50 smithi175 ceph-mon[101434]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:50 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-08-17T17:18:52.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[104516]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:52.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:52.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:52.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[109246]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:52.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:52.599 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:52 smithi175 ceph-mon[101434]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.464 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:18:53.361+0000 7f97afefb640 -1 osd.3 105 log_to_monitors true 2024-08-17T17:18:53.902 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:18:53.501+0000 7f97a7ca5640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[104516]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:53.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-mon[109246]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:18:53.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:53 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:53 smithi175 ceph-mon[101434]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:18:54.179 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:18:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:18:53] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.51.0" 2024-08-17T17:18:54.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:54 smithi175 ceph-mon[101434]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:54 smithi175 ceph-mon[101434]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:54 smithi175 ceph-mon[101434]: osdmap e108: 8 total, 7 up, 8 in 2024-08-17T17:18:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:54 smithi175 ceph-mon[101434]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[104516]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[104516]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[104516]: osdmap e108: 8 total, 7 up, 8 in 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[104516]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[109246]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 251 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[109246]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[109246]: osdmap e108: 8 total, 7 up, 8 in 2024-08-17T17:18:54.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:54 smithi064 ceph-mon[109246]: from='osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]} : dispatch 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413] boot 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[104516]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:18:55.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413] boot 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:55.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:55 smithi064 ceph-mon[109246]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: osd.3 [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413] boot 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:55 smithi175 ceph-mon[101434]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: pgmap v247: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.813 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: Upgrade: Finalizing container_image settings 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:18:56.814 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:18:56.815 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: Upgrade: Complete! 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.816 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: pgmap v247: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: Upgrade: Finalizing container_image settings 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:18:56.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: pgmap v247: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: Upgrade: Finalizing container_image settings 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: Upgrade: Complete! 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:18:56.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: Upgrade: Complete! 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:56.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:56.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:18:56.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:18:56.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:18:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:57 smithi175 ceph-mon[101434]: pgmap v249: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:57.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:57 smithi064 ceph-mon[104516]: pgmap v249: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:57.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:57 smithi064 ceph-mon[109246]: pgmap v249: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-17T17:18:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:58.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:59.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:18:59 smithi175 ceph-mon[101434]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:59.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:18:59 smithi064 ceph-mon[104516]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:18:59.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:18:59 smithi064 ceph-mon[109246]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:00 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-17T17:19:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:00 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:19:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-17T17:19:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:19:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-17T17:19:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:19:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:01 smithi175 ceph-mon[101434]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:01 smithi064 ceph-mon[104516]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:01.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:01 smithi064 ceph-mon[109246]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:03 smithi175 ceph-mon[101434]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:03 smithi064 ceph-mon[104516]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:03 smithi064 ceph-mon[109246]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:04.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:03] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.51.0" 2024-08-17T17:19:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:06 smithi175 ceph-mon[101434]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:06 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:06 smithi064 ceph-mon[104516]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:06 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:06 smithi064 ceph-mon[109246]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:06 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:07.574 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:07.574 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:19:07.584 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-08-17T17:19:07.584 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:19:07.626 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:19:07.626 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:19:07.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:19:07.842 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:08.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:08 smithi064 ceph-mon[104516]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:08.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:08 smithi064 ceph-mon[104516]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:08.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:08 smithi064 ceph-mon[109246]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:08.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:08 smithi064 ceph-mon[109246]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:08 smithi175 ceph-mon[101434]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:08 smithi175 ceph-mon[101434]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:10.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[104516]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:10.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3195440996' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:19:10.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:10.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:10.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[109246]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:10.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3195440996' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:19:10.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:10.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:10.205 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":41,"flags":0,"active_gid":25736,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":3171569814},{"type":"v1","addr":"172.21.15.64:6801","nonce":3171569814}]},"active_addr":"172.21.15.64:6801/3171569814","active_change":"2024-08-17T17:11:24.558790+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25396,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3915252309}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1112427979}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1016971850}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1032548005}]}]} 2024-08-17T17:19:10.209 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:19:10.210 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:19:10.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:19:10.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:10 smithi175 ceph-mon[101434]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3195440996' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:19:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:10.433 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:12.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:12.078 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":110,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:18:55.480484+0000","last_up_change":"2024-08-17T17:18:54.479099+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6803","nonce":2410633150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6805","nonce":2410633150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6809","nonce":2410633150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6807","nonce":2410633150}]},"public_addr":"172.21.15.64:6803/2410633150","cluster_addr":"172.21.15.64:6805/2410633150","heartbeat_back_addr":"172.21.15.64:6809/2410633150","heartbeat_front_addr":"172.21.15.64:6807/2410633150","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6811","nonce":3737445757}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6813","nonce":3737445757}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6817","nonce":3737445757}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6815","nonce":3737445757}]},"public_addr":"172.21.15.64:6811/3737445757","cluster_addr":"172.21.15.64:6813/3737445757","heartbeat_back_addr":"172.21.15.64:6817/3737445757","heartbeat_front_addr":"172.21.15.64:6815/3737445757","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6819","nonce":3328029035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6821","nonce":3328029035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6825","nonce":3328029035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6823","nonce":3328029035}]},"public_addr":"172.21.15.64:6819/3328029035","cluster_addr":"172.21.15.64:6821/3328029035","heartbeat_back_addr":"172.21.15.64:6825/3328029035","heartbeat_front_addr":"172.21.15.64:6823/3328029035","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6827","nonce":2515263413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6829","nonce":2515263413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6833","nonce":2515263413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6831","nonce":2515263413}]},"public_addr":"172.21.15.64:6827/2515263413","cluster_addr":"172.21.15.64:6829/2515263413","heartbeat_back_addr":"172.21.15.64:6833/2515263413","heartbeat_front_addr":"172.21.15.64:6831/2515263413","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.175:0/4035630069":"2024-08-18T17:09:04.985494+0000","172.21.15.175:0/57613042":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/3398196717":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6801/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1135961297":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/1707444063":"2024-08-18T17:09:04.985494+0000","172.21.15.175:6834/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/1031286826":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/4139680086":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/3929894796":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/1358035766":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6800/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/327251329":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/2315698367":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.175:6835/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2821917932":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.175:0/2662913714":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2705070003":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:19:12.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:12 smithi175 ceph-mon[101434]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3470532935' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:12 smithi064 ceph-mon[104516]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3470532935' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:12.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:12 smithi064 ceph-mon[109246]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3470532935' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:12.711 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:19:12.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:19:12.925 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:14.062 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:13] "GET /metrics HTTP/1.1" 200 34151 "" "Prometheus/2.51.0" 2024-08-17T17:19:14.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:14 smithi064 ceph-mon[104516]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:14.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:14 smithi064 ceph-mon[109246]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:14.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:14 smithi175 ceph-mon[101434]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:14.761 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:14.762 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":110,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:18:55.480484+0000","last_up_change":"2024-08-17T17:18:54.479099+0000","last_in_change":"2024-08-17T17:03:26.323680+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6803","nonce":2410633150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6805","nonce":2410633150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6809","nonce":2410633150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6807","nonce":2410633150}]},"public_addr":"172.21.15.64:6803/2410633150","cluster_addr":"172.21.15.64:6805/2410633150","heartbeat_back_addr":"172.21.15.64:6809/2410633150","heartbeat_front_addr":"172.21.15.64:6807/2410633150","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6811","nonce":3737445757}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6813","nonce":3737445757}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6817","nonce":3737445757}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6815","nonce":3737445757}]},"public_addr":"172.21.15.64:6811/3737445757","cluster_addr":"172.21.15.64:6813/3737445757","heartbeat_back_addr":"172.21.15.64:6817/3737445757","heartbeat_front_addr":"172.21.15.64:6815/3737445757","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6819","nonce":3328029035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6821","nonce":3328029035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6825","nonce":3328029035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6823","nonce":3328029035}]},"public_addr":"172.21.15.64:6819/3328029035","cluster_addr":"172.21.15.64:6821/3328029035","heartbeat_back_addr":"172.21.15.64:6825/3328029035","heartbeat_front_addr":"172.21.15.64:6823/3328029035","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6827","nonce":2515263413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6829","nonce":2515263413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6833","nonce":2515263413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6831","nonce":2515263413}]},"public_addr":"172.21.15.64:6827/2515263413","cluster_addr":"172.21.15.64:6829/2515263413","heartbeat_back_addr":"172.21.15.64:6833/2515263413","heartbeat_front_addr":"172.21.15.64:6831/2515263413","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6809","nonce":1423876320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6811","nonce":1423876320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6815","nonce":1423876320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1423876320},{"type":"v1","addr":"172.21.15.175:6813","nonce":1423876320}]},"public_addr":"172.21.15.175:6809/1423876320","cluster_addr":"172.21.15.175:6811/1423876320","heartbeat_back_addr":"172.21.15.175:6815/1423876320","heartbeat_front_addr":"172.21.15.175:6813/1423876320","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6817","nonce":916289961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6819","nonce":916289961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6823","nonce":916289961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":916289961},{"type":"v1","addr":"172.21.15.175:6821","nonce":916289961}]},"public_addr":"172.21.15.175:6817/916289961","cluster_addr":"172.21.15.175:6819/916289961","heartbeat_back_addr":"172.21.15.175:6823/916289961","heartbeat_front_addr":"172.21.15.175:6821/916289961","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6825","nonce":3998219871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6827","nonce":3998219871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6831","nonce":3998219871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":3998219871},{"type":"v1","addr":"172.21.15.175:6829","nonce":3998219871}]},"public_addr":"172.21.15.175:6825/3998219871","cluster_addr":"172.21.15.175:6827/3998219871","heartbeat_back_addr":"172.21.15.175:6831/3998219871","heartbeat_front_addr":"172.21.15.175:6829/3998219871","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/327251329":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6801/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/3929894796":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/4035630069":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1135961297":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/57613042":"2024-08-18T17:09:04.985494+0000","172.21.15.175:0/1707444063":"2024-08-18T17:09:04.985494+0000","172.21.15.175:6834/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/1031286826":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/2705070003":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/4139680086":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6800/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/1358035766":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.175:6835/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2821917932":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.175:0/2662913714":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/3398196717":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.175:0/2315698367":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6801/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:19:15.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3415346503' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:15.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3415346503' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:15.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:19:15.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:19:15.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:19:15.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:19:15.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:19:15.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:19:15.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:19:15.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:19:15.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3415346503' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:19:15.759 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:15.764 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:15.764 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:15.766 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:16 smithi064 ceph-mon[104516]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:16.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:16 smithi064 ceph-mon[109246]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:16 smithi175 ceph-mon[101434]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:16.650 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:16.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:16.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:16.654 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:18.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:18 smithi175 ceph-mon[101434]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:18 smithi064 ceph-mon[104516]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:18.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:18 smithi064 ceph-mon[109246]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:20 smithi175 ceph-mon[101434]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:20.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:20 smithi064 ceph-mon[104516]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:20.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:20 smithi064 ceph-mon[109246]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:22 smithi175 ceph-mon[101434]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:22.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:22 smithi064 ceph-mon[109246]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:22 smithi064 ceph-mon[104516]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:24.032 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:23] "GET /metrics HTTP/1.1" 200 34151 "" "Prometheus/2.51.0" 2024-08-17T17:19:24.293 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:24 smithi064 ceph-mon[104516]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:24.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:24 smithi064 ceph-mon[109246]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:24.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:24 smithi175 ceph-mon[101434]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:25.604 INFO:teuthology.orchestra.run.smithi064.stdout:425201762324 2024-08-17T17:19:25.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:19:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:26 smithi064 ceph-mon[104516]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:26 smithi064 ceph-mon[109246]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:26.563 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:26 smithi175 ceph-mon[101434]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:26.563 INFO:teuthology.orchestra.run.smithi064.stdout:403726925851 2024-08-17T17:19:26.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:19:27.595 INFO:teuthology.orchestra.run.smithi064.stdout:446676598798 2024-08-17T17:19:27.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:19:28.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:28 smithi064 ceph-mon[104516]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:28.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:28 smithi064 ceph-mon[109246]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:28 smithi175 ceph-mon[101434]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:29.096 INFO:teuthology.orchestra.run.smithi064.stdout:236223201471 2024-08-17T17:19:29.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:19:30.054 INFO:teuthology.orchestra.run.smithi064.stdout:468151435272 2024-08-17T17:19:30.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:19:30.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:30 smithi064 ceph-mon[104516]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:30.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:30 smithi064 ceph-mon[109246]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:30 smithi175 ceph-mon[101434]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:31.407 INFO:teuthology.orchestra.run.smithi064.stdout:133143986381 2024-08-17T17:19:31.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:19:31.891 INFO:teuthology.orchestra.run.smithi064.stdout:167503724745 2024-08-17T17:19:31.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:19:31.941 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:31.942 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:32 smithi064 ceph-mon[104516]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:32.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:32 smithi064 ceph-mon[109246]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:32 smithi175 ceph-mon[101434]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:33.437 INFO:teuthology.orchestra.run.smithi064.stdout:201863463109 2024-08-17T17:19:33.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:19:34.172 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:33] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-08-17T17:19:34.280 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:34 smithi064 ceph-mon[104516]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:34.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:34 smithi064 ceph-mon[109246]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:34.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:34 smithi175 ceph-mon[101434]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:36.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:36 smithi064 ceph-mon[104516]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:36.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:36 smithi064 ceph-mon[109246]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:36 smithi175 ceph-mon[101434]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:37.085 INFO:teuthology.orchestra.run.smithi064.stdout:403726925853 2024-08-17T17:19:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2134835965' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:19:37.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2134835965' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:19:37.577 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2134835965' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:19:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:38 smithi064 ceph-mon[104516]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:38.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:38 smithi064 ceph-mon[109246]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:38.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:38 smithi175 ceph-mon[101434]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:38.899 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925851 got 403726925853 for osd.0 2024-08-17T17:19:38.899 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:39.841 INFO:teuthology.orchestra.run.smithi064.stdout:446676598801 2024-08-17T17:19:40.059 INFO:teuthology.orchestra.run.smithi064.stdout:133143986383 2024-08-17T17:19:40.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[104516]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:40.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3749307186' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/626343770' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[109246]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3749307186' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:40.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/626343770' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:19:40.650 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986381 got 133143986383 for osd.4 2024-08-17T17:19:40.651 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:40 smithi175 ceph-mon[101434]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3749307186' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:19:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/626343770' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:19:41.840 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:42.238 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598798 got 446676598801 for osd.2 2024-08-17T17:19:42.238 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:42 smithi175 ceph-mon[101434]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:42.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:42 smithi064 ceph-mon[104516]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:42.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:42 smithi064 ceph-mon[109246]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:44.168 INFO:teuthology.orchestra.run.smithi064.stdout:468151435275 2024-08-17T17:19:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:43] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.51.0" 2024-08-17T17:19:44.208 INFO:teuthology.orchestra.run.smithi064.stdout:425201762328 2024-08-17T17:19:44.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[104516]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:44.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/732432404' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:19:44.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/613524980' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:19:44.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[109246]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:44.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/732432404' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:19:44.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:44 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/613524980' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:19:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:44 smithi175 ceph-mon[101434]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:44 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/732432404' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:19:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:44 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/613524980' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:19:45.270 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435272 got 468151435275 for osd.3 2024-08-17T17:19:45.270 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:46.053 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762324 got 425201762328 for osd.1 2024-08-17T17:19:46.053 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:46.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:46 smithi064 ceph-mon[104516]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:46.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:46 smithi064 ceph-mon[109246]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:46.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:46 smithi175 ceph-mon[101434]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:46.694 INFO:teuthology.orchestra.run.smithi064.stdout:167503724748 2024-08-17T17:19:47.310 INFO:teuthology.orchestra.run.smithi064.stdout:236223201475 2024-08-17T17:19:47.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:47 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/548866039' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:19:47.310 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:47 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/548866039' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:19:47.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724745 got 167503724748 for osd.5 2024-08-17T17:19:47.554 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:47.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:47 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/548866039' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:19:48.135 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201471 got 236223201475 for osd.7 2024-08-17T17:19:48.135 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:48 smithi064 ceph-mon[104516]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4159960829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:19:48.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:48 smithi064 ceph-mon[109246]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:48.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4159960829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:19:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:48 smithi175 ceph-mon[101434]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4159960829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:19:48.840 INFO:teuthology.orchestra.run.smithi064.stdout:201863463113 2024-08-17T17:19:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/902913801' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:19:49.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/902913801' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:19:49.595 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463109 got 201863463113 for osd.6 2024-08-17T17:19:49.596 DEBUG:teuthology.parallel:result is None 2024-08-17T17:19:49.596 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:19:49.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:19:49.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/902913801' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:19:49.813 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:50.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:50 smithi175 ceph-mon[101434]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:50 smithi064 ceph-mon[104516]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:50.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:50 smithi064 ceph-mon[109246]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:51.469 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:51.470 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:19:52.066 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":276,"stamp":"2024-08-17T17:19:50.694403+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":254080,"kb_used_data":23540,"kb_used_omap":34,"kb_used_meta":230365,"kb_avail":749707136,"statfs":{"total":767960285184,"available":767700107264,"internally_reserved":0,"allocated":24104960,"data_stored":19359014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35823,"internal_metadata":235893777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002783"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499047+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:18:55.499047+0000","last_peered":"2024-08-17T17:18:55.499047+0000","last_clean":"2024-08-17T17:18:55.499047+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:18:55.499047+0000","last_undegraded":"2024-08-17T17:18:55.499047+0000","last_fullsized":"2024-08-17T17:18:55.499047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487943+0000","last_change":"2024-08-17T17:18:22.909422+0000","last_active":"2024-08-17T17:18:55.487943+0000","last_peered":"2024-08-17T17:18:55.487943+0000","last_clean":"2024-08-17T17:18:55.487943+0000","last_became_active":"2024-08-17T17:18:22.909017+0000","last_became_peered":"2024-08-17T17:18:22.909017+0000","last_unstale":"2024-08-17T17:18:55.487943+0000","last_undegraded":"2024-08-17T17:18:55.487943+0000","last_fullsized":"2024-08-17T17:18:55.487943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:31.910796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495867+0000","last_change":"2024-08-17T17:18:55.495867+0000","last_active":"2024-08-17T17:18:55.495867+0000","last_peered":"2024-08-17T17:18:55.495867+0000","last_clean":"2024-08-17T17:18:55.495867+0000","last_became_active":"2024-08-17T17:18:55.495425+0000","last_became_peered":"2024-08-17T17:18:55.495425+0000","last_unstale":"2024-08-17T17:18:55.495867+0000","last_undegraded":"2024-08-17T17:18:55.495867+0000","last_fullsized":"2024-08-17T17:18:55.495867+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:35.743221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494262+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:18:55.494262+0000","last_peered":"2024-08-17T17:18:55.494262+0000","last_clean":"2024-08-17T17:18:55.494262+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:18:55.494262+0000","last_undegraded":"2024-08-17T17:18:55.494262+0000","last_fullsized":"2024-08-17T17:18:55.494262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497035+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:18:55.497035+0000","last_peered":"2024-08-17T17:18:55.497035+0000","last_clean":"2024-08-17T17:18:55.497035+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:18:55.497035+0000","last_undegraded":"2024-08-17T17:18:55.497035+0000","last_fullsized":"2024-08-17T17:18:55.497035+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488108+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:18:55.488108+0000","last_peered":"2024-08-17T17:18:55.488108+0000","last_clean":"2024-08-17T17:18:55.488108+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:18:55.488108+0000","last_undegraded":"2024-08-17T17:18:55.488108+0000","last_fullsized":"2024-08-17T17:18:55.488108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495715+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:18:55.495715+0000","last_peered":"2024-08-17T17:18:55.495715+0000","last_clean":"2024-08-17T17:18:55.495715+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:18:55.495715+0000","last_undegraded":"2024-08-17T17:18:55.495715+0000","last_fullsized":"2024-08-17T17:18:55.495715+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493569+0000","last_change":"2024-08-17T17:17:19.404513+0000","last_active":"2024-08-17T17:18:55.493569+0000","last_peered":"2024-08-17T17:18:55.493569+0000","last_clean":"2024-08-17T17:18:55.493569+0000","last_became_active":"2024-08-17T17:17:19.404148+0000","last_became_peered":"2024-08-17T17:17:19.404148+0000","last_unstale":"2024-08-17T17:18:55.493569+0000","last_undegraded":"2024-08-17T17:18:55.493569+0000","last_fullsized":"2024-08-17T17:18:55.493569+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:10.729553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487976+0000","last_change":"2024-08-17T17:17:19.408816+0000","last_active":"2024-08-17T17:18:55.487976+0000","last_peered":"2024-08-17T17:18:55.487976+0000","last_clean":"2024-08-17T17:18:55.487976+0000","last_became_active":"2024-08-17T17:17:19.408449+0000","last_became_peered":"2024-08-17T17:17:19.408449+0000","last_unstale":"2024-08-17T17:18:55.487976+0000","last_undegraded":"2024-08-17T17:18:55.487976+0000","last_fullsized":"2024-08-17T17:18:55.487976+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:20.970954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489725+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:18:54.489725+0000","last_peered":"2024-08-17T17:18:54.489725+0000","last_clean":"2024-08-17T17:18:54.489725+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:18:54.489725+0000","last_undegraded":"2024-08-17T17:18:54.489725+0000","last_fullsized":"2024-08-17T17:18:54.489725+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487651+0000","last_change":"2024-08-17T17:17:50.478014+0000","last_active":"2024-08-17T17:18:55.487651+0000","last_peered":"2024-08-17T17:18:55.487651+0000","last_clean":"2024-08-17T17:18:55.487651+0000","last_became_active":"2024-08-17T17:17:50.477662+0000","last_became_peered":"2024-08-17T17:17:50.477662+0000","last_unstale":"2024-08-17T17:18:55.487651+0000","last_undegraded":"2024-08-17T17:18:55.487651+0000","last_fullsized":"2024-08-17T17:18:55.487651+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:46.006989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498099+0000","last_change":"2024-08-17T17:18:55.498099+0000","last_active":"2024-08-17T17:18:55.498099+0000","last_peered":"2024-08-17T17:18:55.498099+0000","last_clean":"2024-08-17T17:18:55.498099+0000","last_became_active":"2024-08-17T17:18:55.497722+0000","last_became_peered":"2024-08-17T17:18:55.497722+0000","last_unstale":"2024-08-17T17:18:55.498099+0000","last_undegraded":"2024-08-17T17:18:55.498099+0000","last_fullsized":"2024-08-17T17:18:55.498099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:49.290137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487769+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:18:55.487769+0000","last_peered":"2024-08-17T17:18:55.487769+0000","last_clean":"2024-08-17T17:18:55.487769+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:18:55.487769+0000","last_undegraded":"2024-08-17T17:18:55.487769+0000","last_fullsized":"2024-08-17T17:18:55.487769+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486337+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:18:54.486337+0000","last_peered":"2024-08-17T17:18:54.486337+0000","last_clean":"2024-08-17T17:18:54.486337+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:18:54.486337+0000","last_undegraded":"2024-08-17T17:18:54.486337+0000","last_fullsized":"2024-08-17T17:18:54.486337+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497126+0000","last_change":"2024-08-17T17:18:55.496653+0000","last_active":"2024-08-17T17:18:55.497126+0000","last_peered":"2024-08-17T17:18:55.497126+0000","last_clean":"2024-08-17T17:18:55.497126+0000","last_became_active":"2024-08-17T17:18:55.495980+0000","last_became_peered":"2024-08-17T17:18:55.495980+0000","last_unstale":"2024-08-17T17:18:55.497126+0000","last_undegraded":"2024-08-17T17:18:55.497126+0000","last_fullsized":"2024-08-17T17:18:55.497126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:34.698405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484986+0000","last_change":"2024-08-17T17:17:50.472859+0000","last_active":"2024-08-17T17:18:55.484986+0000","last_peered":"2024-08-17T17:18:55.484986+0000","last_clean":"2024-08-17T17:18:55.484986+0000","last_became_active":"2024-08-17T17:17:50.472564+0000","last_became_peered":"2024-08-17T17:17:50.472564+0000","last_unstale":"2024-08-17T17:18:55.484986+0000","last_undegraded":"2024-08-17T17:18:55.484986+0000","last_fullsized":"2024-08-17T17:18:55.484986+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:25.759541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499163+0000","last_change":"2024-08-17T17:18:55.499100+0000","last_active":"2024-08-17T17:18:55.499163+0000","last_peered":"2024-08-17T17:18:55.499163+0000","last_clean":"2024-08-17T17:18:55.499163+0000","last_became_active":"2024-08-17T17:18:55.498395+0000","last_became_peered":"2024-08-17T17:18:55.498395+0000","last_unstale":"2024-08-17T17:18:55.499163+0000","last_undegraded":"2024-08-17T17:18:55.499163+0000","last_fullsized":"2024-08-17T17:18:55.499163+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:19.200301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498811+0000","last_change":"2024-08-17T17:18:55.498039+0000","last_active":"2024-08-17T17:18:55.498811+0000","last_peered":"2024-08-17T17:18:55.498811+0000","last_clean":"2024-08-17T17:18:55.498811+0000","last_became_active":"2024-08-17T17:18:55.497685+0000","last_became_peered":"2024-08-17T17:18:55.497685+0000","last_unstale":"2024-08-17T17:18:55.498811+0000","last_undegraded":"2024-08-17T17:18:55.498811+0000","last_fullsized":"2024-08-17T17:18:55.498811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:49.171022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486783+0000","last_change":"2024-08-17T17:17:50.479775+0000","last_active":"2024-08-17T17:18:54.486783+0000","last_peered":"2024-08-17T17:18:54.486783+0000","last_clean":"2024-08-17T17:18:54.486783+0000","last_became_active":"2024-08-17T17:17:50.479361+0000","last_became_peered":"2024-08-17T17:17:50.479361+0000","last_unstale":"2024-08-17T17:18:54.486783+0000","last_undegraded":"2024-08-17T17:18:54.486783+0000","last_fullsized":"2024-08-17T17:18:54.486783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:52:13.653931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485498+0000","last_change":"2024-08-17T17:18:22.904844+0000","last_active":"2024-08-17T17:18:55.485498+0000","last_peered":"2024-08-17T17:18:55.485498+0000","last_clean":"2024-08-17T17:18:55.485498+0000","last_became_active":"2024-08-17T17:18:22.904512+0000","last_became_peered":"2024-08-17T17:18:22.904512+0000","last_unstale":"2024-08-17T17:18:55.485498+0000","last_undegraded":"2024-08-17T17:18:55.485498+0000","last_fullsized":"2024-08-17T17:18:55.485498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:35.381256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484072+0000","last_change":"2024-08-17T17:18:22.904885+0000","last_active":"2024-08-17T17:18:54.484072+0000","last_peered":"2024-08-17T17:18:54.484072+0000","last_clean":"2024-08-17T17:18:54.484072+0000","last_became_active":"2024-08-17T17:18:22.904535+0000","last_became_peered":"2024-08-17T17:18:22.904535+0000","last_unstale":"2024-08-17T17:18:54.484072+0000","last_undegraded":"2024-08-17T17:18:54.484072+0000","last_fullsized":"2024-08-17T17:18:54.484072+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:00.547235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485431+0000","last_change":"2024-08-17T17:17:50.477509+0000","last_active":"2024-08-17T17:18:55.485431+0000","last_peered":"2024-08-17T17:18:55.485431+0000","last_clean":"2024-08-17T17:18:55.485431+0000","last_became_active":"2024-08-17T17:17:50.477204+0000","last_became_peered":"2024-08-17T17:17:50.477204+0000","last_unstale":"2024-08-17T17:18:55.485431+0000","last_undegraded":"2024-08-17T17:18:55.485431+0000","last_fullsized":"2024-08-17T17:18:55.485431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:02.378683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498012+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:18:55.498012+0000","last_peered":"2024-08-17T17:18:55.498012+0000","last_clean":"2024-08-17T17:18:55.498012+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:18:55.498012+0000","last_undegraded":"2024-08-17T17:18:55.498012+0000","last_fullsized":"2024-08-17T17:18:55.498012+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492775+0000","last_change":"2024-08-17T17:18:55.492713+0000","last_active":"2024-08-17T17:18:55.492775+0000","last_peered":"2024-08-17T17:18:55.492775+0000","last_clean":"2024-08-17T17:18:55.492775+0000","last_became_active":"2024-08-17T17:18:55.492385+0000","last_became_peered":"2024-08-17T17:18:55.492385+0000","last_unstale":"2024-08-17T17:18:55.492775+0000","last_undegraded":"2024-08-17T17:18:55.492775+0000","last_fullsized":"2024-08-17T17:18:55.492775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:33.301910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":401,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486556+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:18:54.486556+0000","last_peered":"2024-08-17T17:18:54.486556+0000","last_clean":"2024-08-17T17:18:54.486556+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:18:54.486556+0000","last_undegraded":"2024-08-17T17:18:54.486556+0000","last_fullsized":"2024-08-17T17:18:54.486556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489011+0000","last_change":"2024-08-17T17:18:22.902347+0000","last_active":"2024-08-17T17:18:55.489011+0000","last_peered":"2024-08-17T17:18:55.489011+0000","last_clean":"2024-08-17T17:18:55.489011+0000","last_became_active":"2024-08-17T17:18:22.901676+0000","last_became_peered":"2024-08-17T17:18:22.901676+0000","last_unstale":"2024-08-17T17:18:55.489011+0000","last_undegraded":"2024-08-17T17:18:55.489011+0000","last_fullsized":"2024-08-17T17:18:55.489011+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:45:20.310108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":829,"num_read_kb":883,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499720+0000","last_change":"2024-08-17T17:18:55.499664+0000","last_active":"2024-08-17T17:18:55.499720+0000","last_peered":"2024-08-17T17:18:55.499720+0000","last_clean":"2024-08-17T17:18:55.499720+0000","last_became_active":"2024-08-17T17:18:55.499216+0000","last_became_peered":"2024-08-17T17:18:55.499216+0000","last_unstale":"2024-08-17T17:18:55.499720+0000","last_undegraded":"2024-08-17T17:18:55.499720+0000","last_fullsized":"2024-08-17T17:18:55.499720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:38.025283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487844+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:18:54.487844+0000","last_peered":"2024-08-17T17:18:54.487844+0000","last_clean":"2024-08-17T17:18:54.487844+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:18:54.487844+0000","last_undegraded":"2024-08-17T17:18:54.487844+0000","last_fullsized":"2024-08-17T17:18:54.487844+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":479,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486454+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:18:54.486454+0000","last_peered":"2024-08-17T17:18:54.486454+0000","last_clean":"2024-08-17T17:18:54.486454+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:18:54.486454+0000","last_undegraded":"2024-08-17T17:18:54.486454+0000","last_fullsized":"2024-08-17T17:18:54.486454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493371+0000","last_change":"2024-08-17T17:18:55.492730+0000","last_active":"2024-08-17T17:18:55.493371+0000","last_peered":"2024-08-17T17:18:55.493371+0000","last_clean":"2024-08-17T17:18:55.493371+0000","last_became_active":"2024-08-17T17:18:55.492439+0000","last_became_peered":"2024-08-17T17:18:55.492439+0000","last_unstale":"2024-08-17T17:18:55.493371+0000","last_undegraded":"2024-08-17T17:18:55.493371+0000","last_fullsized":"2024-08-17T17:18:55.493371+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:01.194689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485444+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:18:55.485444+0000","last_peered":"2024-08-17T17:18:55.485444+0000","last_clean":"2024-08-17T17:18:55.485444+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:18:55.485444+0000","last_undegraded":"2024-08-17T17:18:55.485444+0000","last_fullsized":"2024-08-17T17:18:55.485444+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":474,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495530+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:18:55.495530+0000","last_peered":"2024-08-17T17:18:55.495530+0000","last_clean":"2024-08-17T17:18:55.495530+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:18:55.495530+0000","last_undegraded":"2024-08-17T17:18:55.495530+0000","last_fullsized":"2024-08-17T17:18:55.495530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489396+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:18:55.489396+0000","last_peered":"2024-08-17T17:18:55.489396+0000","last_clean":"2024-08-17T17:18:55.489396+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:18:55.489396+0000","last_undegraded":"2024-08-17T17:18:55.489396+0000","last_fullsized":"2024-08-17T17:18:55.489396+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487933+0000","last_change":"2024-08-17T17:17:50.473987+0000","last_active":"2024-08-17T17:18:55.487933+0000","last_peered":"2024-08-17T17:18:55.487933+0000","last_clean":"2024-08-17T17:18:55.487933+0000","last_became_active":"2024-08-17T17:17:50.472613+0000","last_became_peered":"2024-08-17T17:17:50.472613+0000","last_unstale":"2024-08-17T17:18:55.487933+0000","last_undegraded":"2024-08-17T17:18:55.487933+0000","last_fullsized":"2024-08-17T17:18:55.487933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:35.597393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":975,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489113+0000","last_change":"2024-08-17T17:17:50.473465+0000","last_active":"2024-08-17T17:18:55.489113+0000","last_peered":"2024-08-17T17:18:55.489113+0000","last_clean":"2024-08-17T17:18:55.489113+0000","last_became_active":"2024-08-17T17:17:50.472957+0000","last_became_peered":"2024-08-17T17:17:50.472957+0000","last_unstale":"2024-08-17T17:18:55.489113+0000","last_undegraded":"2024-08-17T17:18:55.489113+0000","last_fullsized":"2024-08-17T17:18:55.489113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:46.964083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1459,"num_read_kb":1465,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484270+0000","last_change":"2024-08-17T17:17:19.403089+0000","last_active":"2024-08-17T17:18:54.484270+0000","last_peered":"2024-08-17T17:18:54.484270+0000","last_clean":"2024-08-17T17:18:54.484270+0000","last_became_active":"2024-08-17T17:17:19.402747+0000","last_became_peered":"2024-08-17T17:17:19.402747+0000","last_unstale":"2024-08-17T17:18:54.484270+0000","last_undegraded":"2024-08-17T17:18:54.484270+0000","last_fullsized":"2024-08-17T17:18:54.484270+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:02.106769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496544+0000","last_change":"2024-08-17T17:18:55.496544+0000","last_active":"2024-08-17T17:18:55.496544+0000","last_peered":"2024-08-17T17:18:55.496544+0000","last_clean":"2024-08-17T17:18:55.496544+0000","last_became_active":"2024-08-17T17:18:55.495961+0000","last_became_peered":"2024-08-17T17:18:55.495961+0000","last_unstale":"2024-08-17T17:18:55.496544+0000","last_undegraded":"2024-08-17T17:18:55.496544+0000","last_fullsized":"2024-08-17T17:18:55.496544+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:43.998720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496478+0000","last_change":"2024-08-17T17:18:55.496478+0000","last_active":"2024-08-17T17:18:55.496478+0000","last_peered":"2024-08-17T17:18:55.496478+0000","last_clean":"2024-08-17T17:18:55.496478+0000","last_became_active":"2024-08-17T17:18:55.496095+0000","last_became_peered":"2024-08-17T17:18:55.496095+0000","last_unstale":"2024-08-17T17:18:55.496478+0000","last_undegraded":"2024-08-17T17:18:55.496478+0000","last_fullsized":"2024-08-17T17:18:55.496478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:46.676023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489743+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:18:55.489743+0000","last_peered":"2024-08-17T17:18:55.489743+0000","last_clean":"2024-08-17T17:18:55.489743+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:18:55.489743+0000","last_undegraded":"2024-08-17T17:18:55.489743+0000","last_fullsized":"2024-08-17T17:18:55.489743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488626+0000","last_change":"2024-08-17T17:17:50.478401+0000","last_active":"2024-08-17T17:18:55.488626+0000","last_peered":"2024-08-17T17:18:55.488626+0000","last_clean":"2024-08-17T17:18:55.488626+0000","last_became_active":"2024-08-17T17:17:50.477120+0000","last_became_peered":"2024-08-17T17:17:50.477120+0000","last_unstale":"2024-08-17T17:18:55.488626+0000","last_undegraded":"2024-08-17T17:18:55.488626+0000","last_fullsized":"2024-08-17T17:18:55.488626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:21.133581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487812+0000","last_change":"2024-08-17T17:18:22.904542+0000","last_active":"2024-08-17T17:18:55.487812+0000","last_peered":"2024-08-17T17:18:55.487812+0000","last_clean":"2024-08-17T17:18:55.487812+0000","last_became_active":"2024-08-17T17:18:22.903998+0000","last_became_peered":"2024-08-17T17:18:22.903998+0000","last_unstale":"2024-08-17T17:18:55.487812+0000","last_undegraded":"2024-08-17T17:18:55.487812+0000","last_fullsized":"2024-08-17T17:18:55.487812+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:14.046789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":583,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487870+0000","last_change":"2024-08-17T17:17:19.409063+0000","last_active":"2024-08-17T17:18:55.487870+0000","last_peered":"2024-08-17T17:18:55.487870+0000","last_clean":"2024-08-17T17:18:55.487870+0000","last_became_active":"2024-08-17T17:17:19.408369+0000","last_became_peered":"2024-08-17T17:17:19.408369+0000","last_unstale":"2024-08-17T17:18:55.487870+0000","last_undegraded":"2024-08-17T17:18:55.487870+0000","last_fullsized":"2024-08-17T17:18:55.487870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:23.298610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484914+0000","last_change":"2024-08-17T17:17:19.406225+0000","last_active":"2024-08-17T17:18:55.484914+0000","last_peered":"2024-08-17T17:18:55.484914+0000","last_clean":"2024-08-17T17:18:55.484914+0000","last_became_active":"2024-08-17T17:17:19.405856+0000","last_became_peered":"2024-08-17T17:17:19.405856+0000","last_unstale":"2024-08-17T17:18:55.484914+0000","last_undegraded":"2024-08-17T17:18:55.484914+0000","last_fullsized":"2024-08-17T17:18:55.484914+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:09.665590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498816+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:18:55.498816+0000","last_peered":"2024-08-17T17:18:55.498816+0000","last_clean":"2024-08-17T17:18:55.498816+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:18:55.498816+0000","last_undegraded":"2024-08-17T17:18:55.498816+0000","last_fullsized":"2024-08-17T17:18:55.498816+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":522,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487987+0000","last_change":"2024-08-17T17:17:50.477255+0000","last_active":"2024-08-17T17:18:55.487987+0000","last_peered":"2024-08-17T17:18:55.487987+0000","last_clean":"2024-08-17T17:18:55.487987+0000","last_became_active":"2024-08-17T17:17:50.476780+0000","last_became_peered":"2024-08-17T17:17:50.476780+0000","last_unstale":"2024-08-17T17:18:55.487987+0000","last_undegraded":"2024-08-17T17:18:55.487987+0000","last_fullsized":"2024-08-17T17:18:55.487987+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:19.520906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497382+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:18:55.497382+0000","last_peered":"2024-08-17T17:18:55.497382+0000","last_clean":"2024-08-17T17:18:55.497382+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:18:55.497382+0000","last_undegraded":"2024-08-17T17:18:55.497382+0000","last_fullsized":"2024-08-17T17:18:55.497382+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484980+0000","last_change":"2024-08-17T17:17:50.478316+0000","last_active":"2024-08-17T17:18:55.484980+0000","last_peered":"2024-08-17T17:18:55.484980+0000","last_clean":"2024-08-17T17:18:55.484980+0000","last_became_active":"2024-08-17T17:17:50.477739+0000","last_became_peered":"2024-08-17T17:17:50.477739+0000","last_unstale":"2024-08-17T17:18:55.484980+0000","last_undegraded":"2024-08-17T17:18:55.484980+0000","last_fullsized":"2024-08-17T17:18:55.484980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:49.649407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":351,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493908+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:18:55.493908+0000","last_peered":"2024-08-17T17:18:55.493908+0000","last_clean":"2024-08-17T17:18:55.493908+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:18:55.493908+0000","last_undegraded":"2024-08-17T17:18:55.493908+0000","last_fullsized":"2024-08-17T17:18:55.493908+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482264+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:18:54.482264+0000","last_peered":"2024-08-17T17:18:54.482264+0000","last_clean":"2024-08-17T17:18:54.482264+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:18:54.482264+0000","last_undegraded":"2024-08-17T17:18:54.482264+0000","last_fullsized":"2024-08-17T17:18:54.482264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485723+0000","last_change":"2024-08-17T17:17:50.477504+0000","last_active":"2024-08-17T17:18:55.485723+0000","last_peered":"2024-08-17T17:18:55.485723+0000","last_clean":"2024-08-17T17:18:55.485723+0000","last_became_active":"2024-08-17T17:17:50.477172+0000","last_became_peered":"2024-08-17T17:17:50.477172+0000","last_unstale":"2024-08-17T17:18:55.485723+0000","last_undegraded":"2024-08-17T17:18:55.485723+0000","last_fullsized":"2024-08-17T17:18:55.485723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:19.990582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485166+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:18:55.485166+0000","last_peered":"2024-08-17T17:18:55.485166+0000","last_clean":"2024-08-17T17:18:55.485166+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:18:55.485166+0000","last_undegraded":"2024-08-17T17:18:55.485166+0000","last_fullsized":"2024-08-17T17:18:55.485166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488468+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:18:55.488468+0000","last_peered":"2024-08-17T17:18:55.488468+0000","last_clean":"2024-08-17T17:18:55.488468+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:18:55.488468+0000","last_undegraded":"2024-08-17T17:18:55.488468+0000","last_fullsized":"2024-08-17T17:18:55.488468+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495009+0000","last_change":"2024-08-17T17:18:55.495009+0000","last_active":"2024-08-17T17:18:55.495009+0000","last_peered":"2024-08-17T17:18:55.495009+0000","last_clean":"2024-08-17T17:18:55.495009+0000","last_became_active":"2024-08-17T17:18:55.494532+0000","last_became_peered":"2024-08-17T17:18:55.494532+0000","last_unstale":"2024-08-17T17:18:55.495009+0000","last_undegraded":"2024-08-17T17:18:55.495009+0000","last_fullsized":"2024-08-17T17:18:55.495009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:54.695395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489852+0000","last_change":"2024-08-17T17:17:50.474706+0000","last_active":"2024-08-17T17:18:54.489852+0000","last_peered":"2024-08-17T17:18:54.489852+0000","last_clean":"2024-08-17T17:18:54.489852+0000","last_became_active":"2024-08-17T17:17:50.474392+0000","last_became_peered":"2024-08-17T17:17:50.474392+0000","last_unstale":"2024-08-17T17:18:54.489852+0000","last_undegraded":"2024-08-17T17:18:54.489852+0000","last_fullsized":"2024-08-17T17:18:54.489852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:38.744232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498410+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:18:55.498410+0000","last_peered":"2024-08-17T17:18:55.498410+0000","last_clean":"2024-08-17T17:18:55.498410+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:18:55.498410+0000","last_undegraded":"2024-08-17T17:18:55.498410+0000","last_fullsized":"2024-08-17T17:18:55.498410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494317+0000","last_change":"2024-08-17T17:18:55.493741+0000","last_active":"2024-08-17T17:18:55.494317+0000","last_peered":"2024-08-17T17:18:55.494317+0000","last_clean":"2024-08-17T17:18:55.494317+0000","last_became_active":"2024-08-17T17:18:55.493423+0000","last_became_peered":"2024-08-17T17:18:55.493423+0000","last_unstale":"2024-08-17T17:18:55.494317+0000","last_undegraded":"2024-08-17T17:18:55.494317+0000","last_fullsized":"2024-08-17T17:18:55.494317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:15.411411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497350+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:18:55.497350+0000","last_peered":"2024-08-17T17:18:55.497350+0000","last_clean":"2024-08-17T17:18:55.497350+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:18:55.497350+0000","last_undegraded":"2024-08-17T17:18:55.497350+0000","last_fullsized":"2024-08-17T17:18:55.497350+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487419+0000","last_change":"2024-08-17T17:17:50.474242+0000","last_active":"2024-08-17T17:18:54.487419+0000","last_peered":"2024-08-17T17:18:54.487419+0000","last_clean":"2024-08-17T17:18:54.487419+0000","last_became_active":"2024-08-17T17:17:50.473923+0000","last_became_peered":"2024-08-17T17:17:50.473923+0000","last_unstale":"2024-08-17T17:18:54.487419+0000","last_undegraded":"2024-08-17T17:18:54.487419+0000","last_fullsized":"2024-08-17T17:18:54.487419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:13.899933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493594+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:18:55.493594+0000","last_peered":"2024-08-17T17:18:55.493594+0000","last_clean":"2024-08-17T17:18:55.493594+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:18:55.493594+0000","last_undegraded":"2024-08-17T17:18:55.493594+0000","last_fullsized":"2024-08-17T17:18:55.493594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499603+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:18:55.499603+0000","last_peered":"2024-08-17T17:18:55.499603+0000","last_clean":"2024-08-17T17:18:55.499603+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:18:55.499603+0000","last_undegraded":"2024-08-17T17:18:55.499603+0000","last_fullsized":"2024-08-17T17:18:55.499603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495398+0000","last_change":"2024-08-17T17:18:55.495340+0000","last_active":"2024-08-17T17:18:55.495398+0000","last_peered":"2024-08-17T17:18:55.495398+0000","last_clean":"2024-08-17T17:18:55.495398+0000","last_became_active":"2024-08-17T17:18:55.495057+0000","last_became_peered":"2024-08-17T17:18:55.495057+0000","last_unstale":"2024-08-17T17:18:55.495398+0000","last_undegraded":"2024-08-17T17:18:55.495398+0000","last_fullsized":"2024-08-17T17:18:55.495398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:26.993423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498393+0000","last_change":"2024-08-17T17:18:55.498330+0000","last_active":"2024-08-17T17:18:55.498393+0000","last_peered":"2024-08-17T17:18:55.498393+0000","last_clean":"2024-08-17T17:18:55.498393+0000","last_became_active":"2024-08-17T17:18:55.496964+0000","last_became_peered":"2024-08-17T17:18:55.496964+0000","last_unstale":"2024-08-17T17:18:55.498393+0000","last_undegraded":"2024-08-17T17:18:55.498393+0000","last_fullsized":"2024-08-17T17:18:55.498393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:09:56.482828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482472+0000","last_change":"2024-08-17T17:18:22.904972+0000","last_active":"2024-08-17T17:18:54.482472+0000","last_peered":"2024-08-17T17:18:54.482472+0000","last_clean":"2024-08-17T17:18:54.482472+0000","last_became_active":"2024-08-17T17:18:22.904630+0000","last_became_peered":"2024-08-17T17:18:22.904630+0000","last_unstale":"2024-08-17T17:18:54.482472+0000","last_undegraded":"2024-08-17T17:18:54.482472+0000","last_fullsized":"2024-08-17T17:18:54.482472+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:01.926349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494019+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:18:55.494019+0000","last_peered":"2024-08-17T17:18:55.494019+0000","last_clean":"2024-08-17T17:18:55.494019+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:18:55.494019+0000","last_undegraded":"2024-08-17T17:18:55.494019+0000","last_fullsized":"2024-08-17T17:18:55.494019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484803+0000","last_change":"2024-08-17T17:17:19.406837+0000","last_active":"2024-08-17T17:18:55.484803+0000","last_peered":"2024-08-17T17:18:55.484803+0000","last_clean":"2024-08-17T17:18:55.484803+0000","last_became_active":"2024-08-17T17:17:19.406413+0000","last_became_peered":"2024-08-17T17:17:19.406413+0000","last_unstale":"2024-08-17T17:18:55.484803+0000","last_undegraded":"2024-08-17T17:18:55.484803+0000","last_fullsized":"2024-08-17T17:18:55.484803+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:22.497997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498573+0000","last_change":"2024-08-17T17:18:55.498573+0000","last_active":"2024-08-17T17:18:55.498573+0000","last_peered":"2024-08-17T17:18:55.498573+0000","last_clean":"2024-08-17T17:18:55.498573+0000","last_became_active":"2024-08-17T17:18:55.498230+0000","last_became_peered":"2024-08-17T17:18:55.498230+0000","last_unstale":"2024-08-17T17:18:55.498573+0000","last_undegraded":"2024-08-17T17:18:55.498573+0000","last_fullsized":"2024-08-17T17:18:55.498573+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.161705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.491060+0000","last_change":"2024-08-17T17:17:50.472168+0000","last_active":"2024-08-17T17:18:54.491060+0000","last_peered":"2024-08-17T17:18:54.491060+0000","last_clean":"2024-08-17T17:18:54.491060+0000","last_became_active":"2024-08-17T17:17:50.471864+0000","last_became_peered":"2024-08-17T17:17:50.471864+0000","last_unstale":"2024-08-17T17:18:54.491060+0000","last_undegraded":"2024-08-17T17:18:54.491060+0000","last_fullsized":"2024-08-17T17:18:54.491060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:32.866769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492171+0000","last_change":"2024-08-17T17:18:55.492171+0000","last_active":"2024-08-17T17:18:55.492171+0000","last_peered":"2024-08-17T17:18:55.492171+0000","last_clean":"2024-08-17T17:18:55.492171+0000","last_became_active":"2024-08-17T17:18:55.491790+0000","last_became_peered":"2024-08-17T17:18:55.491790+0000","last_unstale":"2024-08-17T17:18:55.492171+0000","last_undegraded":"2024-08-17T17:18:55.492171+0000","last_fullsized":"2024-08-17T17:18:55.492171+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:44:21.883054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.490901+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:18:54.490901+0000","last_peered":"2024-08-17T17:18:54.490901+0000","last_clean":"2024-08-17T17:18:54.490901+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:18:54.490901+0000","last_undegraded":"2024-08-17T17:18:54.490901+0000","last_fullsized":"2024-08-17T17:18:54.490901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.500485+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:18:55.500485+0000","last_peered":"2024-08-17T17:18:55.500485+0000","last_clean":"2024-08-17T17:18:55.500485+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:18:55.500485+0000","last_undegraded":"2024-08-17T17:18:55.500485+0000","last_fullsized":"2024-08-17T17:18:55.500485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498299+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:18:55.498299+0000","last_peered":"2024-08-17T17:18:55.498299+0000","last_clean":"2024-08-17T17:18:55.498299+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:18:55.498299+0000","last_undegraded":"2024-08-17T17:18:55.498299+0000","last_fullsized":"2024-08-17T17:18:55.498299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482239+0000","last_change":"2024-08-17T17:18:22.903634+0000","last_active":"2024-08-17T17:18:54.482239+0000","last_peered":"2024-08-17T17:18:54.482239+0000","last_clean":"2024-08-17T17:18:54.482239+0000","last_became_active":"2024-08-17T17:18:22.903279+0000","last_became_peered":"2024-08-17T17:18:22.903279+0000","last_unstale":"2024-08-17T17:18:54.482239+0000","last_undegraded":"2024-08-17T17:18:54.482239+0000","last_fullsized":"2024-08-17T17:18:54.482239+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:08.309006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.483800+0000","last_change":"2024-08-17T17:18:22.902105+0000","last_active":"2024-08-17T17:18:55.483800+0000","last_peered":"2024-08-17T17:18:55.483800+0000","last_clean":"2024-08-17T17:18:55.483800+0000","last_became_active":"2024-08-17T17:18:22.901714+0000","last_became_peered":"2024-08-17T17:18:22.901714+0000","last_unstale":"2024-08-17T17:18:55.483800+0000","last_undegraded":"2024-08-17T17:18:55.483800+0000","last_fullsized":"2024-08-17T17:18:55.483800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:33.515070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201476,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36996,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708156,"statfs":{"total":95995035648,"available":95957151744,"internally_reserved":0,"allocated":6021120,"data_stored":5413201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463114,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1772,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1814528,"data_stored":1218505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724750,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32308,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712844,"statfs":{"total":95995035648,"available":95961952256,"internally_reserved":0,"allocated":1216512,"data_stored":624228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986386,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":5296,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":5423104,"data_stored":4822771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1180,"kb_used_omap":5,"kb_used_meta":27514,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1208320,"data_stored":623667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":28174409},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1212416,"data_stored":624193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":28172461},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1176,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1204224,"data_stored":623323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":28172705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925856,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6004736,"data_stored":5409126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":28172892},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:19:52.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:19:52.283 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:52.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:52 smithi064 ceph-mon[104516]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:52.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:52 smithi064 ceph-mon[104516]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:52.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:52 smithi064 ceph-mon[109246]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:52.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:52 smithi064 ceph-mon[109246]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:52.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:52 smithi175 ceph-mon[101434]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:52 smithi175 ceph-mon[101434]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:54.062 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:54.062 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:19:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:19:53] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.51.0" 2024-08-17T17:19:54.064 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:19:54.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:54 smithi064 ceph-mon[104516]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:54.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:54 smithi064 ceph-mon[109246]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:54 smithi175 ceph-mon[101434]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:54.814 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2024-08-17T17:19:52.694845+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":254080,"kb_used_data":23540,"kb_used_omap":34,"kb_used_meta":230365,"kb_avail":749707136,"statfs":{"total":767960285184,"available":767700107264,"internally_reserved":0,"allocated":24104960,"data_stored":19359014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35823,"internal_metadata":235893777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002813"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499047+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:18:55.499047+0000","last_peered":"2024-08-17T17:18:55.499047+0000","last_clean":"2024-08-17T17:18:55.499047+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:18:55.499047+0000","last_undegraded":"2024-08-17T17:18:55.499047+0000","last_fullsized":"2024-08-17T17:18:55.499047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487943+0000","last_change":"2024-08-17T17:18:22.909422+0000","last_active":"2024-08-17T17:18:55.487943+0000","last_peered":"2024-08-17T17:18:55.487943+0000","last_clean":"2024-08-17T17:18:55.487943+0000","last_became_active":"2024-08-17T17:18:22.909017+0000","last_became_peered":"2024-08-17T17:18:22.909017+0000","last_unstale":"2024-08-17T17:18:55.487943+0000","last_undegraded":"2024-08-17T17:18:55.487943+0000","last_fullsized":"2024-08-17T17:18:55.487943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:31.910796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495867+0000","last_change":"2024-08-17T17:18:55.495867+0000","last_active":"2024-08-17T17:18:55.495867+0000","last_peered":"2024-08-17T17:18:55.495867+0000","last_clean":"2024-08-17T17:18:55.495867+0000","last_became_active":"2024-08-17T17:18:55.495425+0000","last_became_peered":"2024-08-17T17:18:55.495425+0000","last_unstale":"2024-08-17T17:18:55.495867+0000","last_undegraded":"2024-08-17T17:18:55.495867+0000","last_fullsized":"2024-08-17T17:18:55.495867+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:35.743221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494262+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:18:55.494262+0000","last_peered":"2024-08-17T17:18:55.494262+0000","last_clean":"2024-08-17T17:18:55.494262+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:18:55.494262+0000","last_undegraded":"2024-08-17T17:18:55.494262+0000","last_fullsized":"2024-08-17T17:18:55.494262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497035+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:18:55.497035+0000","last_peered":"2024-08-17T17:18:55.497035+0000","last_clean":"2024-08-17T17:18:55.497035+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:18:55.497035+0000","last_undegraded":"2024-08-17T17:18:55.497035+0000","last_fullsized":"2024-08-17T17:18:55.497035+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488108+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:18:55.488108+0000","last_peered":"2024-08-17T17:18:55.488108+0000","last_clean":"2024-08-17T17:18:55.488108+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:18:55.488108+0000","last_undegraded":"2024-08-17T17:18:55.488108+0000","last_fullsized":"2024-08-17T17:18:55.488108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495715+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:18:55.495715+0000","last_peered":"2024-08-17T17:18:55.495715+0000","last_clean":"2024-08-17T17:18:55.495715+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:18:55.495715+0000","last_undegraded":"2024-08-17T17:18:55.495715+0000","last_fullsized":"2024-08-17T17:18:55.495715+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493569+0000","last_change":"2024-08-17T17:17:19.404513+0000","last_active":"2024-08-17T17:18:55.493569+0000","last_peered":"2024-08-17T17:18:55.493569+0000","last_clean":"2024-08-17T17:18:55.493569+0000","last_became_active":"2024-08-17T17:17:19.404148+0000","last_became_peered":"2024-08-17T17:17:19.404148+0000","last_unstale":"2024-08-17T17:18:55.493569+0000","last_undegraded":"2024-08-17T17:18:55.493569+0000","last_fullsized":"2024-08-17T17:18:55.493569+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:10.729553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487976+0000","last_change":"2024-08-17T17:17:19.408816+0000","last_active":"2024-08-17T17:18:55.487976+0000","last_peered":"2024-08-17T17:18:55.487976+0000","last_clean":"2024-08-17T17:18:55.487976+0000","last_became_active":"2024-08-17T17:17:19.408449+0000","last_became_peered":"2024-08-17T17:17:19.408449+0000","last_unstale":"2024-08-17T17:18:55.487976+0000","last_undegraded":"2024-08-17T17:18:55.487976+0000","last_fullsized":"2024-08-17T17:18:55.487976+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:20.970954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489725+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:18:54.489725+0000","last_peered":"2024-08-17T17:18:54.489725+0000","last_clean":"2024-08-17T17:18:54.489725+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:18:54.489725+0000","last_undegraded":"2024-08-17T17:18:54.489725+0000","last_fullsized":"2024-08-17T17:18:54.489725+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487651+0000","last_change":"2024-08-17T17:17:50.478014+0000","last_active":"2024-08-17T17:18:55.487651+0000","last_peered":"2024-08-17T17:18:55.487651+0000","last_clean":"2024-08-17T17:18:55.487651+0000","last_became_active":"2024-08-17T17:17:50.477662+0000","last_became_peered":"2024-08-17T17:17:50.477662+0000","last_unstale":"2024-08-17T17:18:55.487651+0000","last_undegraded":"2024-08-17T17:18:55.487651+0000","last_fullsized":"2024-08-17T17:18:55.487651+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:46.006989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498099+0000","last_change":"2024-08-17T17:18:55.498099+0000","last_active":"2024-08-17T17:18:55.498099+0000","last_peered":"2024-08-17T17:18:55.498099+0000","last_clean":"2024-08-17T17:18:55.498099+0000","last_became_active":"2024-08-17T17:18:55.497722+0000","last_became_peered":"2024-08-17T17:18:55.497722+0000","last_unstale":"2024-08-17T17:18:55.498099+0000","last_undegraded":"2024-08-17T17:18:55.498099+0000","last_fullsized":"2024-08-17T17:18:55.498099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:49.290137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487769+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:18:55.487769+0000","last_peered":"2024-08-17T17:18:55.487769+0000","last_clean":"2024-08-17T17:18:55.487769+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:18:55.487769+0000","last_undegraded":"2024-08-17T17:18:55.487769+0000","last_fullsized":"2024-08-17T17:18:55.487769+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486337+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:18:54.486337+0000","last_peered":"2024-08-17T17:18:54.486337+0000","last_clean":"2024-08-17T17:18:54.486337+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:18:54.486337+0000","last_undegraded":"2024-08-17T17:18:54.486337+0000","last_fullsized":"2024-08-17T17:18:54.486337+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497126+0000","last_change":"2024-08-17T17:18:55.496653+0000","last_active":"2024-08-17T17:18:55.497126+0000","last_peered":"2024-08-17T17:18:55.497126+0000","last_clean":"2024-08-17T17:18:55.497126+0000","last_became_active":"2024-08-17T17:18:55.495980+0000","last_became_peered":"2024-08-17T17:18:55.495980+0000","last_unstale":"2024-08-17T17:18:55.497126+0000","last_undegraded":"2024-08-17T17:18:55.497126+0000","last_fullsized":"2024-08-17T17:18:55.497126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:34.698405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484986+0000","last_change":"2024-08-17T17:17:50.472859+0000","last_active":"2024-08-17T17:18:55.484986+0000","last_peered":"2024-08-17T17:18:55.484986+0000","last_clean":"2024-08-17T17:18:55.484986+0000","last_became_active":"2024-08-17T17:17:50.472564+0000","last_became_peered":"2024-08-17T17:17:50.472564+0000","last_unstale":"2024-08-17T17:18:55.484986+0000","last_undegraded":"2024-08-17T17:18:55.484986+0000","last_fullsized":"2024-08-17T17:18:55.484986+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:25.759541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499163+0000","last_change":"2024-08-17T17:18:55.499100+0000","last_active":"2024-08-17T17:18:55.499163+0000","last_peered":"2024-08-17T17:18:55.499163+0000","last_clean":"2024-08-17T17:18:55.499163+0000","last_became_active":"2024-08-17T17:18:55.498395+0000","last_became_peered":"2024-08-17T17:18:55.498395+0000","last_unstale":"2024-08-17T17:18:55.499163+0000","last_undegraded":"2024-08-17T17:18:55.499163+0000","last_fullsized":"2024-08-17T17:18:55.499163+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:19.200301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498811+0000","last_change":"2024-08-17T17:18:55.498039+0000","last_active":"2024-08-17T17:18:55.498811+0000","last_peered":"2024-08-17T17:18:55.498811+0000","last_clean":"2024-08-17T17:18:55.498811+0000","last_became_active":"2024-08-17T17:18:55.497685+0000","last_became_peered":"2024-08-17T17:18:55.497685+0000","last_unstale":"2024-08-17T17:18:55.498811+0000","last_undegraded":"2024-08-17T17:18:55.498811+0000","last_fullsized":"2024-08-17T17:18:55.498811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:49.171022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486783+0000","last_change":"2024-08-17T17:17:50.479775+0000","last_active":"2024-08-17T17:18:54.486783+0000","last_peered":"2024-08-17T17:18:54.486783+0000","last_clean":"2024-08-17T17:18:54.486783+0000","last_became_active":"2024-08-17T17:17:50.479361+0000","last_became_peered":"2024-08-17T17:17:50.479361+0000","last_unstale":"2024-08-17T17:18:54.486783+0000","last_undegraded":"2024-08-17T17:18:54.486783+0000","last_fullsized":"2024-08-17T17:18:54.486783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:52:13.653931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485498+0000","last_change":"2024-08-17T17:18:22.904844+0000","last_active":"2024-08-17T17:18:55.485498+0000","last_peered":"2024-08-17T17:18:55.485498+0000","last_clean":"2024-08-17T17:18:55.485498+0000","last_became_active":"2024-08-17T17:18:22.904512+0000","last_became_peered":"2024-08-17T17:18:22.904512+0000","last_unstale":"2024-08-17T17:18:55.485498+0000","last_undegraded":"2024-08-17T17:18:55.485498+0000","last_fullsized":"2024-08-17T17:18:55.485498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:35.381256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484072+0000","last_change":"2024-08-17T17:18:22.904885+0000","last_active":"2024-08-17T17:18:54.484072+0000","last_peered":"2024-08-17T17:18:54.484072+0000","last_clean":"2024-08-17T17:18:54.484072+0000","last_became_active":"2024-08-17T17:18:22.904535+0000","last_became_peered":"2024-08-17T17:18:22.904535+0000","last_unstale":"2024-08-17T17:18:54.484072+0000","last_undegraded":"2024-08-17T17:18:54.484072+0000","last_fullsized":"2024-08-17T17:18:54.484072+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:00.547235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485431+0000","last_change":"2024-08-17T17:17:50.477509+0000","last_active":"2024-08-17T17:18:55.485431+0000","last_peered":"2024-08-17T17:18:55.485431+0000","last_clean":"2024-08-17T17:18:55.485431+0000","last_became_active":"2024-08-17T17:17:50.477204+0000","last_became_peered":"2024-08-17T17:17:50.477204+0000","last_unstale":"2024-08-17T17:18:55.485431+0000","last_undegraded":"2024-08-17T17:18:55.485431+0000","last_fullsized":"2024-08-17T17:18:55.485431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:02.378683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498012+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:18:55.498012+0000","last_peered":"2024-08-17T17:18:55.498012+0000","last_clean":"2024-08-17T17:18:55.498012+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:18:55.498012+0000","last_undegraded":"2024-08-17T17:18:55.498012+0000","last_fullsized":"2024-08-17T17:18:55.498012+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492775+0000","last_change":"2024-08-17T17:18:55.492713+0000","last_active":"2024-08-17T17:18:55.492775+0000","last_peered":"2024-08-17T17:18:55.492775+0000","last_clean":"2024-08-17T17:18:55.492775+0000","last_became_active":"2024-08-17T17:18:55.492385+0000","last_became_peered":"2024-08-17T17:18:55.492385+0000","last_unstale":"2024-08-17T17:18:55.492775+0000","last_undegraded":"2024-08-17T17:18:55.492775+0000","last_fullsized":"2024-08-17T17:18:55.492775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:33.301910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":401,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486556+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:18:54.486556+0000","last_peered":"2024-08-17T17:18:54.486556+0000","last_clean":"2024-08-17T17:18:54.486556+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:18:54.486556+0000","last_undegraded":"2024-08-17T17:18:54.486556+0000","last_fullsized":"2024-08-17T17:18:54.486556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489011+0000","last_change":"2024-08-17T17:18:22.902347+0000","last_active":"2024-08-17T17:18:55.489011+0000","last_peered":"2024-08-17T17:18:55.489011+0000","last_clean":"2024-08-17T17:18:55.489011+0000","last_became_active":"2024-08-17T17:18:22.901676+0000","last_became_peered":"2024-08-17T17:18:22.901676+0000","last_unstale":"2024-08-17T17:18:55.489011+0000","last_undegraded":"2024-08-17T17:18:55.489011+0000","last_fullsized":"2024-08-17T17:18:55.489011+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:45:20.310108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":829,"num_read_kb":883,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499720+0000","last_change":"2024-08-17T17:18:55.499664+0000","last_active":"2024-08-17T17:18:55.499720+0000","last_peered":"2024-08-17T17:18:55.499720+0000","last_clean":"2024-08-17T17:18:55.499720+0000","last_became_active":"2024-08-17T17:18:55.499216+0000","last_became_peered":"2024-08-17T17:18:55.499216+0000","last_unstale":"2024-08-17T17:18:55.499720+0000","last_undegraded":"2024-08-17T17:18:55.499720+0000","last_fullsized":"2024-08-17T17:18:55.499720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:38.025283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487844+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:18:54.487844+0000","last_peered":"2024-08-17T17:18:54.487844+0000","last_clean":"2024-08-17T17:18:54.487844+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:18:54.487844+0000","last_undegraded":"2024-08-17T17:18:54.487844+0000","last_fullsized":"2024-08-17T17:18:54.487844+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":479,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486454+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:18:54.486454+0000","last_peered":"2024-08-17T17:18:54.486454+0000","last_clean":"2024-08-17T17:18:54.486454+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:18:54.486454+0000","last_undegraded":"2024-08-17T17:18:54.486454+0000","last_fullsized":"2024-08-17T17:18:54.486454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493371+0000","last_change":"2024-08-17T17:18:55.492730+0000","last_active":"2024-08-17T17:18:55.493371+0000","last_peered":"2024-08-17T17:18:55.493371+0000","last_clean":"2024-08-17T17:18:55.493371+0000","last_became_active":"2024-08-17T17:18:55.492439+0000","last_became_peered":"2024-08-17T17:18:55.492439+0000","last_unstale":"2024-08-17T17:18:55.493371+0000","last_undegraded":"2024-08-17T17:18:55.493371+0000","last_fullsized":"2024-08-17T17:18:55.493371+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:01.194689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485444+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:18:55.485444+0000","last_peered":"2024-08-17T17:18:55.485444+0000","last_clean":"2024-08-17T17:18:55.485444+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:18:55.485444+0000","last_undegraded":"2024-08-17T17:18:55.485444+0000","last_fullsized":"2024-08-17T17:18:55.485444+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":474,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495530+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:18:55.495530+0000","last_peered":"2024-08-17T17:18:55.495530+0000","last_clean":"2024-08-17T17:18:55.495530+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:18:55.495530+0000","last_undegraded":"2024-08-17T17:18:55.495530+0000","last_fullsized":"2024-08-17T17:18:55.495530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489396+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:18:55.489396+0000","last_peered":"2024-08-17T17:18:55.489396+0000","last_clean":"2024-08-17T17:18:55.489396+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:18:55.489396+0000","last_undegraded":"2024-08-17T17:18:55.489396+0000","last_fullsized":"2024-08-17T17:18:55.489396+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487933+0000","last_change":"2024-08-17T17:17:50.473987+0000","last_active":"2024-08-17T17:18:55.487933+0000","last_peered":"2024-08-17T17:18:55.487933+0000","last_clean":"2024-08-17T17:18:55.487933+0000","last_became_active":"2024-08-17T17:17:50.472613+0000","last_became_peered":"2024-08-17T17:17:50.472613+0000","last_unstale":"2024-08-17T17:18:55.487933+0000","last_undegraded":"2024-08-17T17:18:55.487933+0000","last_fullsized":"2024-08-17T17:18:55.487933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:35.597393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":975,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489113+0000","last_change":"2024-08-17T17:17:50.473465+0000","last_active":"2024-08-17T17:18:55.489113+0000","last_peered":"2024-08-17T17:18:55.489113+0000","last_clean":"2024-08-17T17:18:55.489113+0000","last_became_active":"2024-08-17T17:17:50.472957+0000","last_became_peered":"2024-08-17T17:17:50.472957+0000","last_unstale":"2024-08-17T17:18:55.489113+0000","last_undegraded":"2024-08-17T17:18:55.489113+0000","last_fullsized":"2024-08-17T17:18:55.489113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:46.964083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1459,"num_read_kb":1465,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484270+0000","last_change":"2024-08-17T17:17:19.403089+0000","last_active":"2024-08-17T17:18:54.484270+0000","last_peered":"2024-08-17T17:18:54.484270+0000","last_clean":"2024-08-17T17:18:54.484270+0000","last_became_active":"2024-08-17T17:17:19.402747+0000","last_became_peered":"2024-08-17T17:17:19.402747+0000","last_unstale":"2024-08-17T17:18:54.484270+0000","last_undegraded":"2024-08-17T17:18:54.484270+0000","last_fullsized":"2024-08-17T17:18:54.484270+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:02.106769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496544+0000","last_change":"2024-08-17T17:18:55.496544+0000","last_active":"2024-08-17T17:18:55.496544+0000","last_peered":"2024-08-17T17:18:55.496544+0000","last_clean":"2024-08-17T17:18:55.496544+0000","last_became_active":"2024-08-17T17:18:55.495961+0000","last_became_peered":"2024-08-17T17:18:55.495961+0000","last_unstale":"2024-08-17T17:18:55.496544+0000","last_undegraded":"2024-08-17T17:18:55.496544+0000","last_fullsized":"2024-08-17T17:18:55.496544+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:43.998720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496478+0000","last_change":"2024-08-17T17:18:55.496478+0000","last_active":"2024-08-17T17:18:55.496478+0000","last_peered":"2024-08-17T17:18:55.496478+0000","last_clean":"2024-08-17T17:18:55.496478+0000","last_became_active":"2024-08-17T17:18:55.496095+0000","last_became_peered":"2024-08-17T17:18:55.496095+0000","last_unstale":"2024-08-17T17:18:55.496478+0000","last_undegraded":"2024-08-17T17:18:55.496478+0000","last_fullsized":"2024-08-17T17:18:55.496478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:46.676023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489743+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:18:55.489743+0000","last_peered":"2024-08-17T17:18:55.489743+0000","last_clean":"2024-08-17T17:18:55.489743+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:18:55.489743+0000","last_undegraded":"2024-08-17T17:18:55.489743+0000","last_fullsized":"2024-08-17T17:18:55.489743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488626+0000","last_change":"2024-08-17T17:17:50.478401+0000","last_active":"2024-08-17T17:18:55.488626+0000","last_peered":"2024-08-17T17:18:55.488626+0000","last_clean":"2024-08-17T17:18:55.488626+0000","last_became_active":"2024-08-17T17:17:50.477120+0000","last_became_peered":"2024-08-17T17:17:50.477120+0000","last_unstale":"2024-08-17T17:18:55.488626+0000","last_undegraded":"2024-08-17T17:18:55.488626+0000","last_fullsized":"2024-08-17T17:18:55.488626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:21.133581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487812+0000","last_change":"2024-08-17T17:18:22.904542+0000","last_active":"2024-08-17T17:18:55.487812+0000","last_peered":"2024-08-17T17:18:55.487812+0000","last_clean":"2024-08-17T17:18:55.487812+0000","last_became_active":"2024-08-17T17:18:22.903998+0000","last_became_peered":"2024-08-17T17:18:22.903998+0000","last_unstale":"2024-08-17T17:18:55.487812+0000","last_undegraded":"2024-08-17T17:18:55.487812+0000","last_fullsized":"2024-08-17T17:18:55.487812+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:14.046789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":583,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487870+0000","last_change":"2024-08-17T17:17:19.409063+0000","last_active":"2024-08-17T17:18:55.487870+0000","last_peered":"2024-08-17T17:18:55.487870+0000","last_clean":"2024-08-17T17:18:55.487870+0000","last_became_active":"2024-08-17T17:17:19.408369+0000","last_became_peered":"2024-08-17T17:17:19.408369+0000","last_unstale":"2024-08-17T17:18:55.487870+0000","last_undegraded":"2024-08-17T17:18:55.487870+0000","last_fullsized":"2024-08-17T17:18:55.487870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:23.298610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484914+0000","last_change":"2024-08-17T17:17:19.406225+0000","last_active":"2024-08-17T17:18:55.484914+0000","last_peered":"2024-08-17T17:18:55.484914+0000","last_clean":"2024-08-17T17:18:55.484914+0000","last_became_active":"2024-08-17T17:17:19.405856+0000","last_became_peered":"2024-08-17T17:17:19.405856+0000","last_unstale":"2024-08-17T17:18:55.484914+0000","last_undegraded":"2024-08-17T17:18:55.484914+0000","last_fullsized":"2024-08-17T17:18:55.484914+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:09.665590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498816+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:18:55.498816+0000","last_peered":"2024-08-17T17:18:55.498816+0000","last_clean":"2024-08-17T17:18:55.498816+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:18:55.498816+0000","last_undegraded":"2024-08-17T17:18:55.498816+0000","last_fullsized":"2024-08-17T17:18:55.498816+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":522,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487987+0000","last_change":"2024-08-17T17:17:50.477255+0000","last_active":"2024-08-17T17:18:55.487987+0000","last_peered":"2024-08-17T17:18:55.487987+0000","last_clean":"2024-08-17T17:18:55.487987+0000","last_became_active":"2024-08-17T17:17:50.476780+0000","last_became_peered":"2024-08-17T17:17:50.476780+0000","last_unstale":"2024-08-17T17:18:55.487987+0000","last_undegraded":"2024-08-17T17:18:55.487987+0000","last_fullsized":"2024-08-17T17:18:55.487987+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:19.520906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497382+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:18:55.497382+0000","last_peered":"2024-08-17T17:18:55.497382+0000","last_clean":"2024-08-17T17:18:55.497382+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:18:55.497382+0000","last_undegraded":"2024-08-17T17:18:55.497382+0000","last_fullsized":"2024-08-17T17:18:55.497382+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484980+0000","last_change":"2024-08-17T17:17:50.478316+0000","last_active":"2024-08-17T17:18:55.484980+0000","last_peered":"2024-08-17T17:18:55.484980+0000","last_clean":"2024-08-17T17:18:55.484980+0000","last_became_active":"2024-08-17T17:17:50.477739+0000","last_became_peered":"2024-08-17T17:17:50.477739+0000","last_unstale":"2024-08-17T17:18:55.484980+0000","last_undegraded":"2024-08-17T17:18:55.484980+0000","last_fullsized":"2024-08-17T17:18:55.484980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:49.649407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":351,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493908+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:18:55.493908+0000","last_peered":"2024-08-17T17:18:55.493908+0000","last_clean":"2024-08-17T17:18:55.493908+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:18:55.493908+0000","last_undegraded":"2024-08-17T17:18:55.493908+0000","last_fullsized":"2024-08-17T17:18:55.493908+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482264+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:18:54.482264+0000","last_peered":"2024-08-17T17:18:54.482264+0000","last_clean":"2024-08-17T17:18:54.482264+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:18:54.482264+0000","last_undegraded":"2024-08-17T17:18:54.482264+0000","last_fullsized":"2024-08-17T17:18:54.482264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485723+0000","last_change":"2024-08-17T17:17:50.477504+0000","last_active":"2024-08-17T17:18:55.485723+0000","last_peered":"2024-08-17T17:18:55.485723+0000","last_clean":"2024-08-17T17:18:55.485723+0000","last_became_active":"2024-08-17T17:17:50.477172+0000","last_became_peered":"2024-08-17T17:17:50.477172+0000","last_unstale":"2024-08-17T17:18:55.485723+0000","last_undegraded":"2024-08-17T17:18:55.485723+0000","last_fullsized":"2024-08-17T17:18:55.485723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:19.990582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485166+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:18:55.485166+0000","last_peered":"2024-08-17T17:18:55.485166+0000","last_clean":"2024-08-17T17:18:55.485166+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:18:55.485166+0000","last_undegraded":"2024-08-17T17:18:55.485166+0000","last_fullsized":"2024-08-17T17:18:55.485166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488468+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:18:55.488468+0000","last_peered":"2024-08-17T17:18:55.488468+0000","last_clean":"2024-08-17T17:18:55.488468+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:18:55.488468+0000","last_undegraded":"2024-08-17T17:18:55.488468+0000","last_fullsized":"2024-08-17T17:18:55.488468+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495009+0000","last_change":"2024-08-17T17:18:55.495009+0000","last_active":"2024-08-17T17:18:55.495009+0000","last_peered":"2024-08-17T17:18:55.495009+0000","last_clean":"2024-08-17T17:18:55.495009+0000","last_became_active":"2024-08-17T17:18:55.494532+0000","last_became_peered":"2024-08-17T17:18:55.494532+0000","last_unstale":"2024-08-17T17:18:55.495009+0000","last_undegraded":"2024-08-17T17:18:55.495009+0000","last_fullsized":"2024-08-17T17:18:55.495009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:54.695395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489852+0000","last_change":"2024-08-17T17:17:50.474706+0000","last_active":"2024-08-17T17:18:54.489852+0000","last_peered":"2024-08-17T17:18:54.489852+0000","last_clean":"2024-08-17T17:18:54.489852+0000","last_became_active":"2024-08-17T17:17:50.474392+0000","last_became_peered":"2024-08-17T17:17:50.474392+0000","last_unstale":"2024-08-17T17:18:54.489852+0000","last_undegraded":"2024-08-17T17:18:54.489852+0000","last_fullsized":"2024-08-17T17:18:54.489852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:38.744232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498410+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:18:55.498410+0000","last_peered":"2024-08-17T17:18:55.498410+0000","last_clean":"2024-08-17T17:18:55.498410+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:18:55.498410+0000","last_undegraded":"2024-08-17T17:18:55.498410+0000","last_fullsized":"2024-08-17T17:18:55.498410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494317+0000","last_change":"2024-08-17T17:18:55.493741+0000","last_active":"2024-08-17T17:18:55.494317+0000","last_peered":"2024-08-17T17:18:55.494317+0000","last_clean":"2024-08-17T17:18:55.494317+0000","last_became_active":"2024-08-17T17:18:55.493423+0000","last_became_peered":"2024-08-17T17:18:55.493423+0000","last_unstale":"2024-08-17T17:18:55.494317+0000","last_undegraded":"2024-08-17T17:18:55.494317+0000","last_fullsized":"2024-08-17T17:18:55.494317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:15.411411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497350+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:18:55.497350+0000","last_peered":"2024-08-17T17:18:55.497350+0000","last_clean":"2024-08-17T17:18:55.497350+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:18:55.497350+0000","last_undegraded":"2024-08-17T17:18:55.497350+0000","last_fullsized":"2024-08-17T17:18:55.497350+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487419+0000","last_change":"2024-08-17T17:17:50.474242+0000","last_active":"2024-08-17T17:18:54.487419+0000","last_peered":"2024-08-17T17:18:54.487419+0000","last_clean":"2024-08-17T17:18:54.487419+0000","last_became_active":"2024-08-17T17:17:50.473923+0000","last_became_peered":"2024-08-17T17:17:50.473923+0000","last_unstale":"2024-08-17T17:18:54.487419+0000","last_undegraded":"2024-08-17T17:18:54.487419+0000","last_fullsized":"2024-08-17T17:18:54.487419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:13.899933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493594+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:18:55.493594+0000","last_peered":"2024-08-17T17:18:55.493594+0000","last_clean":"2024-08-17T17:18:55.493594+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:18:55.493594+0000","last_undegraded":"2024-08-17T17:18:55.493594+0000","last_fullsized":"2024-08-17T17:18:55.493594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499603+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:18:55.499603+0000","last_peered":"2024-08-17T17:18:55.499603+0000","last_clean":"2024-08-17T17:18:55.499603+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:18:55.499603+0000","last_undegraded":"2024-08-17T17:18:55.499603+0000","last_fullsized":"2024-08-17T17:18:55.499603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495398+0000","last_change":"2024-08-17T17:18:55.495340+0000","last_active":"2024-08-17T17:18:55.495398+0000","last_peered":"2024-08-17T17:18:55.495398+0000","last_clean":"2024-08-17T17:18:55.495398+0000","last_became_active":"2024-08-17T17:18:55.495057+0000","last_became_peered":"2024-08-17T17:18:55.495057+0000","last_unstale":"2024-08-17T17:18:55.495398+0000","last_undegraded":"2024-08-17T17:18:55.495398+0000","last_fullsized":"2024-08-17T17:18:55.495398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:26.993423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498393+0000","last_change":"2024-08-17T17:18:55.498330+0000","last_active":"2024-08-17T17:18:55.498393+0000","last_peered":"2024-08-17T17:18:55.498393+0000","last_clean":"2024-08-17T17:18:55.498393+0000","last_became_active":"2024-08-17T17:18:55.496964+0000","last_became_peered":"2024-08-17T17:18:55.496964+0000","last_unstale":"2024-08-17T17:18:55.498393+0000","last_undegraded":"2024-08-17T17:18:55.498393+0000","last_fullsized":"2024-08-17T17:18:55.498393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:09:56.482828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482472+0000","last_change":"2024-08-17T17:18:22.904972+0000","last_active":"2024-08-17T17:18:54.482472+0000","last_peered":"2024-08-17T17:18:54.482472+0000","last_clean":"2024-08-17T17:18:54.482472+0000","last_became_active":"2024-08-17T17:18:22.904630+0000","last_became_peered":"2024-08-17T17:18:22.904630+0000","last_unstale":"2024-08-17T17:18:54.482472+0000","last_undegraded":"2024-08-17T17:18:54.482472+0000","last_fullsized":"2024-08-17T17:18:54.482472+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:01.926349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494019+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:18:55.494019+0000","last_peered":"2024-08-17T17:18:55.494019+0000","last_clean":"2024-08-17T17:18:55.494019+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:18:55.494019+0000","last_undegraded":"2024-08-17T17:18:55.494019+0000","last_fullsized":"2024-08-17T17:18:55.494019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484803+0000","last_change":"2024-08-17T17:17:19.406837+0000","last_active":"2024-08-17T17:18:55.484803+0000","last_peered":"2024-08-17T17:18:55.484803+0000","last_clean":"2024-08-17T17:18:55.484803+0000","last_became_active":"2024-08-17T17:17:19.406413+0000","last_became_peered":"2024-08-17T17:17:19.406413+0000","last_unstale":"2024-08-17T17:18:55.484803+0000","last_undegraded":"2024-08-17T17:18:55.484803+0000","last_fullsized":"2024-08-17T17:18:55.484803+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:22.497997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498573+0000","last_change":"2024-08-17T17:18:55.498573+0000","last_active":"2024-08-17T17:18:55.498573+0000","last_peered":"2024-08-17T17:18:55.498573+0000","last_clean":"2024-08-17T17:18:55.498573+0000","last_became_active":"2024-08-17T17:18:55.498230+0000","last_became_peered":"2024-08-17T17:18:55.498230+0000","last_unstale":"2024-08-17T17:18:55.498573+0000","last_undegraded":"2024-08-17T17:18:55.498573+0000","last_fullsized":"2024-08-17T17:18:55.498573+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.161705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.491060+0000","last_change":"2024-08-17T17:17:50.472168+0000","last_active":"2024-08-17T17:18:54.491060+0000","last_peered":"2024-08-17T17:18:54.491060+0000","last_clean":"2024-08-17T17:18:54.491060+0000","last_became_active":"2024-08-17T17:17:50.471864+0000","last_became_peered":"2024-08-17T17:17:50.471864+0000","last_unstale":"2024-08-17T17:18:54.491060+0000","last_undegraded":"2024-08-17T17:18:54.491060+0000","last_fullsized":"2024-08-17T17:18:54.491060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:32.866769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492171+0000","last_change":"2024-08-17T17:18:55.492171+0000","last_active":"2024-08-17T17:18:55.492171+0000","last_peered":"2024-08-17T17:18:55.492171+0000","last_clean":"2024-08-17T17:18:55.492171+0000","last_became_active":"2024-08-17T17:18:55.491790+0000","last_became_peered":"2024-08-17T17:18:55.491790+0000","last_unstale":"2024-08-17T17:18:55.492171+0000","last_undegraded":"2024-08-17T17:18:55.492171+0000","last_fullsized":"2024-08-17T17:18:55.492171+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:44:21.883054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.490901+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:18:54.490901+0000","last_peered":"2024-08-17T17:18:54.490901+0000","last_clean":"2024-08-17T17:18:54.490901+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:18:54.490901+0000","last_undegraded":"2024-08-17T17:18:54.490901+0000","last_fullsized":"2024-08-17T17:18:54.490901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.500485+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:18:55.500485+0000","last_peered":"2024-08-17T17:18:55.500485+0000","last_clean":"2024-08-17T17:18:55.500485+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:18:55.500485+0000","last_undegraded":"2024-08-17T17:18:55.500485+0000","last_fullsized":"2024-08-17T17:18:55.500485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498299+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:18:55.498299+0000","last_peered":"2024-08-17T17:18:55.498299+0000","last_clean":"2024-08-17T17:18:55.498299+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:18:55.498299+0000","last_undegraded":"2024-08-17T17:18:55.498299+0000","last_fullsized":"2024-08-17T17:18:55.498299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482239+0000","last_change":"2024-08-17T17:18:22.903634+0000","last_active":"2024-08-17T17:18:54.482239+0000","last_peered":"2024-08-17T17:18:54.482239+0000","last_clean":"2024-08-17T17:18:54.482239+0000","last_became_active":"2024-08-17T17:18:22.903279+0000","last_became_peered":"2024-08-17T17:18:22.903279+0000","last_unstale":"2024-08-17T17:18:54.482239+0000","last_undegraded":"2024-08-17T17:18:54.482239+0000","last_fullsized":"2024-08-17T17:18:54.482239+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:08.309006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.483800+0000","last_change":"2024-08-17T17:18:22.902105+0000","last_active":"2024-08-17T17:18:55.483800+0000","last_peered":"2024-08-17T17:18:55.483800+0000","last_clean":"2024-08-17T17:18:55.483800+0000","last_became_active":"2024-08-17T17:18:22.901714+0000","last_became_peered":"2024-08-17T17:18:22.901714+0000","last_unstale":"2024-08-17T17:18:55.483800+0000","last_undegraded":"2024-08-17T17:18:55.483800+0000","last_fullsized":"2024-08-17T17:18:55.483800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:33.515070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201476,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36996,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708156,"statfs":{"total":95995035648,"available":95957151744,"internally_reserved":0,"allocated":6021120,"data_stored":5413201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463114,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1772,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1814528,"data_stored":1218505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724750,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32308,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712844,"statfs":{"total":95995035648,"available":95961952256,"internally_reserved":0,"allocated":1216512,"data_stored":624228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986386,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":5296,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":5423104,"data_stored":4822771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1180,"kb_used_omap":5,"kb_used_meta":27514,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1208320,"data_stored":623667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":28174409},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1212416,"data_stored":624193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":28172461},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1176,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1204224,"data_stored":623323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":28172705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925857,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6004736,"data_stored":5409126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":28172892},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:19:54.817 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:19:54.818 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:19:54.818 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:19:54.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:55 smithi064 ceph-mon[104516]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:55 smithi064 ceph-mon[109246]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:55 smithi175 ceph-mon[101434]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:56.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:56 smithi064 ceph-mon[104516]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:56.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:19:56.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:56 smithi064 ceph-mon[109246]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:56.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:19:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:56 smithi175 ceph-mon[101434]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:19:56.878 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:56.878 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:19:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1463999159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:19:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1463999159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:19:57.753 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:19:57.754 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:19:57.765 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-08-17T17:19:57.765 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:19:57.805 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:19:57.805 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:19:57.805 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-08-17T17:19:57.807 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:19:57.808 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '91f88e4d132ac08dc53355fa3d9717a53236a3a9'}} 2024-08-17T17:19:57.808 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:19:57.808 INFO:tasks.thrashosds:Beginning thrashosds... 2024-08-17T17:19:57.808 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:19:57.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:19:57.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:19:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1463999159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:19:58.203 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:58.604 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:19:58 smithi175 ceph-mon[101434]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:58.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:19:58 smithi064 ceph-mon[104516]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:58.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:19:58 smithi064 ceph-mon[109246]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:19:59.956 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:19:59.957 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:20:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:00 smithi175 ceph-mon[101434]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:00 smithi175 ceph-mon[101434]: overall HEALTH_OK 2024-08-17T17:20:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[104516]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[104516]: overall HEALTH_OK 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[109246]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[109246]: overall HEALTH_OK 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:01.318 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2024-08-17T17:19:58.696199+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":254080,"kb_used_data":23540,"kb_used_omap":34,"kb_used_meta":230365,"kb_avail":749707136,"statfs":{"total":767960285184,"available":767700107264,"internally_reserved":0,"allocated":24104960,"data_stored":19359014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35823,"internal_metadata":235893777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002665"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499047+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:18:55.499047+0000","last_peered":"2024-08-17T17:18:55.499047+0000","last_clean":"2024-08-17T17:18:55.499047+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:18:55.499047+0000","last_undegraded":"2024-08-17T17:18:55.499047+0000","last_fullsized":"2024-08-17T17:18:55.499047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487943+0000","last_change":"2024-08-17T17:18:22.909422+0000","last_active":"2024-08-17T17:18:55.487943+0000","last_peered":"2024-08-17T17:18:55.487943+0000","last_clean":"2024-08-17T17:18:55.487943+0000","last_became_active":"2024-08-17T17:18:22.909017+0000","last_became_peered":"2024-08-17T17:18:22.909017+0000","last_unstale":"2024-08-17T17:18:55.487943+0000","last_undegraded":"2024-08-17T17:18:55.487943+0000","last_fullsized":"2024-08-17T17:18:55.487943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:31.910796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495867+0000","last_change":"2024-08-17T17:18:55.495867+0000","last_active":"2024-08-17T17:18:55.495867+0000","last_peered":"2024-08-17T17:18:55.495867+0000","last_clean":"2024-08-17T17:18:55.495867+0000","last_became_active":"2024-08-17T17:18:55.495425+0000","last_became_peered":"2024-08-17T17:18:55.495425+0000","last_unstale":"2024-08-17T17:18:55.495867+0000","last_undegraded":"2024-08-17T17:18:55.495867+0000","last_fullsized":"2024-08-17T17:18:55.495867+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:35.743221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494262+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:18:55.494262+0000","last_peered":"2024-08-17T17:18:55.494262+0000","last_clean":"2024-08-17T17:18:55.494262+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:18:55.494262+0000","last_undegraded":"2024-08-17T17:18:55.494262+0000","last_fullsized":"2024-08-17T17:18:55.494262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497035+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:18:55.497035+0000","last_peered":"2024-08-17T17:18:55.497035+0000","last_clean":"2024-08-17T17:18:55.497035+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:18:55.497035+0000","last_undegraded":"2024-08-17T17:18:55.497035+0000","last_fullsized":"2024-08-17T17:18:55.497035+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488108+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:18:55.488108+0000","last_peered":"2024-08-17T17:18:55.488108+0000","last_clean":"2024-08-17T17:18:55.488108+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:18:55.488108+0000","last_undegraded":"2024-08-17T17:18:55.488108+0000","last_fullsized":"2024-08-17T17:18:55.488108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495715+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:18:55.495715+0000","last_peered":"2024-08-17T17:18:55.495715+0000","last_clean":"2024-08-17T17:18:55.495715+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:18:55.495715+0000","last_undegraded":"2024-08-17T17:18:55.495715+0000","last_fullsized":"2024-08-17T17:18:55.495715+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493569+0000","last_change":"2024-08-17T17:17:19.404513+0000","last_active":"2024-08-17T17:18:55.493569+0000","last_peered":"2024-08-17T17:18:55.493569+0000","last_clean":"2024-08-17T17:18:55.493569+0000","last_became_active":"2024-08-17T17:17:19.404148+0000","last_became_peered":"2024-08-17T17:17:19.404148+0000","last_unstale":"2024-08-17T17:18:55.493569+0000","last_undegraded":"2024-08-17T17:18:55.493569+0000","last_fullsized":"2024-08-17T17:18:55.493569+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:10.729553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487976+0000","last_change":"2024-08-17T17:17:19.408816+0000","last_active":"2024-08-17T17:18:55.487976+0000","last_peered":"2024-08-17T17:18:55.487976+0000","last_clean":"2024-08-17T17:18:55.487976+0000","last_became_active":"2024-08-17T17:17:19.408449+0000","last_became_peered":"2024-08-17T17:17:19.408449+0000","last_unstale":"2024-08-17T17:18:55.487976+0000","last_undegraded":"2024-08-17T17:18:55.487976+0000","last_fullsized":"2024-08-17T17:18:55.487976+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:20.970954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489725+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:18:54.489725+0000","last_peered":"2024-08-17T17:18:54.489725+0000","last_clean":"2024-08-17T17:18:54.489725+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:18:54.489725+0000","last_undegraded":"2024-08-17T17:18:54.489725+0000","last_fullsized":"2024-08-17T17:18:54.489725+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487651+0000","last_change":"2024-08-17T17:17:50.478014+0000","last_active":"2024-08-17T17:18:55.487651+0000","last_peered":"2024-08-17T17:18:55.487651+0000","last_clean":"2024-08-17T17:18:55.487651+0000","last_became_active":"2024-08-17T17:17:50.477662+0000","last_became_peered":"2024-08-17T17:17:50.477662+0000","last_unstale":"2024-08-17T17:18:55.487651+0000","last_undegraded":"2024-08-17T17:18:55.487651+0000","last_fullsized":"2024-08-17T17:18:55.487651+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:46.006989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498099+0000","last_change":"2024-08-17T17:18:55.498099+0000","last_active":"2024-08-17T17:18:55.498099+0000","last_peered":"2024-08-17T17:18:55.498099+0000","last_clean":"2024-08-17T17:18:55.498099+0000","last_became_active":"2024-08-17T17:18:55.497722+0000","last_became_peered":"2024-08-17T17:18:55.497722+0000","last_unstale":"2024-08-17T17:18:55.498099+0000","last_undegraded":"2024-08-17T17:18:55.498099+0000","last_fullsized":"2024-08-17T17:18:55.498099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:49.290137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487769+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:18:55.487769+0000","last_peered":"2024-08-17T17:18:55.487769+0000","last_clean":"2024-08-17T17:18:55.487769+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:18:55.487769+0000","last_undegraded":"2024-08-17T17:18:55.487769+0000","last_fullsized":"2024-08-17T17:18:55.487769+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486337+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:18:54.486337+0000","last_peered":"2024-08-17T17:18:54.486337+0000","last_clean":"2024-08-17T17:18:54.486337+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:18:54.486337+0000","last_undegraded":"2024-08-17T17:18:54.486337+0000","last_fullsized":"2024-08-17T17:18:54.486337+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497126+0000","last_change":"2024-08-17T17:18:55.496653+0000","last_active":"2024-08-17T17:18:55.497126+0000","last_peered":"2024-08-17T17:18:55.497126+0000","last_clean":"2024-08-17T17:18:55.497126+0000","last_became_active":"2024-08-17T17:18:55.495980+0000","last_became_peered":"2024-08-17T17:18:55.495980+0000","last_unstale":"2024-08-17T17:18:55.497126+0000","last_undegraded":"2024-08-17T17:18:55.497126+0000","last_fullsized":"2024-08-17T17:18:55.497126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:34.698405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484986+0000","last_change":"2024-08-17T17:17:50.472859+0000","last_active":"2024-08-17T17:18:55.484986+0000","last_peered":"2024-08-17T17:18:55.484986+0000","last_clean":"2024-08-17T17:18:55.484986+0000","last_became_active":"2024-08-17T17:17:50.472564+0000","last_became_peered":"2024-08-17T17:17:50.472564+0000","last_unstale":"2024-08-17T17:18:55.484986+0000","last_undegraded":"2024-08-17T17:18:55.484986+0000","last_fullsized":"2024-08-17T17:18:55.484986+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:25.759541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499163+0000","last_change":"2024-08-17T17:18:55.499100+0000","last_active":"2024-08-17T17:18:55.499163+0000","last_peered":"2024-08-17T17:18:55.499163+0000","last_clean":"2024-08-17T17:18:55.499163+0000","last_became_active":"2024-08-17T17:18:55.498395+0000","last_became_peered":"2024-08-17T17:18:55.498395+0000","last_unstale":"2024-08-17T17:18:55.499163+0000","last_undegraded":"2024-08-17T17:18:55.499163+0000","last_fullsized":"2024-08-17T17:18:55.499163+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:19.200301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498811+0000","last_change":"2024-08-17T17:18:55.498039+0000","last_active":"2024-08-17T17:18:55.498811+0000","last_peered":"2024-08-17T17:18:55.498811+0000","last_clean":"2024-08-17T17:18:55.498811+0000","last_became_active":"2024-08-17T17:18:55.497685+0000","last_became_peered":"2024-08-17T17:18:55.497685+0000","last_unstale":"2024-08-17T17:18:55.498811+0000","last_undegraded":"2024-08-17T17:18:55.498811+0000","last_fullsized":"2024-08-17T17:18:55.498811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:49.171022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486783+0000","last_change":"2024-08-17T17:17:50.479775+0000","last_active":"2024-08-17T17:18:54.486783+0000","last_peered":"2024-08-17T17:18:54.486783+0000","last_clean":"2024-08-17T17:18:54.486783+0000","last_became_active":"2024-08-17T17:17:50.479361+0000","last_became_peered":"2024-08-17T17:17:50.479361+0000","last_unstale":"2024-08-17T17:18:54.486783+0000","last_undegraded":"2024-08-17T17:18:54.486783+0000","last_fullsized":"2024-08-17T17:18:54.486783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:52:13.653931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485498+0000","last_change":"2024-08-17T17:18:22.904844+0000","last_active":"2024-08-17T17:18:55.485498+0000","last_peered":"2024-08-17T17:18:55.485498+0000","last_clean":"2024-08-17T17:18:55.485498+0000","last_became_active":"2024-08-17T17:18:22.904512+0000","last_became_peered":"2024-08-17T17:18:22.904512+0000","last_unstale":"2024-08-17T17:18:55.485498+0000","last_undegraded":"2024-08-17T17:18:55.485498+0000","last_fullsized":"2024-08-17T17:18:55.485498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:35.381256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484072+0000","last_change":"2024-08-17T17:18:22.904885+0000","last_active":"2024-08-17T17:18:54.484072+0000","last_peered":"2024-08-17T17:18:54.484072+0000","last_clean":"2024-08-17T17:18:54.484072+0000","last_became_active":"2024-08-17T17:18:22.904535+0000","last_became_peered":"2024-08-17T17:18:22.904535+0000","last_unstale":"2024-08-17T17:18:54.484072+0000","last_undegraded":"2024-08-17T17:18:54.484072+0000","last_fullsized":"2024-08-17T17:18:54.484072+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:00.547235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485431+0000","last_change":"2024-08-17T17:17:50.477509+0000","last_active":"2024-08-17T17:18:55.485431+0000","last_peered":"2024-08-17T17:18:55.485431+0000","last_clean":"2024-08-17T17:18:55.485431+0000","last_became_active":"2024-08-17T17:17:50.477204+0000","last_became_peered":"2024-08-17T17:17:50.477204+0000","last_unstale":"2024-08-17T17:18:55.485431+0000","last_undegraded":"2024-08-17T17:18:55.485431+0000","last_fullsized":"2024-08-17T17:18:55.485431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:02.378683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498012+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:18:55.498012+0000","last_peered":"2024-08-17T17:18:55.498012+0000","last_clean":"2024-08-17T17:18:55.498012+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:18:55.498012+0000","last_undegraded":"2024-08-17T17:18:55.498012+0000","last_fullsized":"2024-08-17T17:18:55.498012+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492775+0000","last_change":"2024-08-17T17:18:55.492713+0000","last_active":"2024-08-17T17:18:55.492775+0000","last_peered":"2024-08-17T17:18:55.492775+0000","last_clean":"2024-08-17T17:18:55.492775+0000","last_became_active":"2024-08-17T17:18:55.492385+0000","last_became_peered":"2024-08-17T17:18:55.492385+0000","last_unstale":"2024-08-17T17:18:55.492775+0000","last_undegraded":"2024-08-17T17:18:55.492775+0000","last_fullsized":"2024-08-17T17:18:55.492775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:33.301910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":401,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486556+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:18:54.486556+0000","last_peered":"2024-08-17T17:18:54.486556+0000","last_clean":"2024-08-17T17:18:54.486556+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:18:54.486556+0000","last_undegraded":"2024-08-17T17:18:54.486556+0000","last_fullsized":"2024-08-17T17:18:54.486556+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489011+0000","last_change":"2024-08-17T17:18:22.902347+0000","last_active":"2024-08-17T17:18:55.489011+0000","last_peered":"2024-08-17T17:18:55.489011+0000","last_clean":"2024-08-17T17:18:55.489011+0000","last_became_active":"2024-08-17T17:18:22.901676+0000","last_became_peered":"2024-08-17T17:18:22.901676+0000","last_unstale":"2024-08-17T17:18:55.489011+0000","last_undegraded":"2024-08-17T17:18:55.489011+0000","last_fullsized":"2024-08-17T17:18:55.489011+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:45:20.310108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":829,"num_read_kb":883,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499720+0000","last_change":"2024-08-17T17:18:55.499664+0000","last_active":"2024-08-17T17:18:55.499720+0000","last_peered":"2024-08-17T17:18:55.499720+0000","last_clean":"2024-08-17T17:18:55.499720+0000","last_became_active":"2024-08-17T17:18:55.499216+0000","last_became_peered":"2024-08-17T17:18:55.499216+0000","last_unstale":"2024-08-17T17:18:55.499720+0000","last_undegraded":"2024-08-17T17:18:55.499720+0000","last_fullsized":"2024-08-17T17:18:55.499720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:38.025283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487844+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:18:54.487844+0000","last_peered":"2024-08-17T17:18:54.487844+0000","last_clean":"2024-08-17T17:18:54.487844+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:18:54.487844+0000","last_undegraded":"2024-08-17T17:18:54.487844+0000","last_fullsized":"2024-08-17T17:18:54.487844+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":479,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486454+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:18:54.486454+0000","last_peered":"2024-08-17T17:18:54.486454+0000","last_clean":"2024-08-17T17:18:54.486454+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:18:54.486454+0000","last_undegraded":"2024-08-17T17:18:54.486454+0000","last_fullsized":"2024-08-17T17:18:54.486454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493371+0000","last_change":"2024-08-17T17:18:55.492730+0000","last_active":"2024-08-17T17:18:55.493371+0000","last_peered":"2024-08-17T17:18:55.493371+0000","last_clean":"2024-08-17T17:18:55.493371+0000","last_became_active":"2024-08-17T17:18:55.492439+0000","last_became_peered":"2024-08-17T17:18:55.492439+0000","last_unstale":"2024-08-17T17:18:55.493371+0000","last_undegraded":"2024-08-17T17:18:55.493371+0000","last_fullsized":"2024-08-17T17:18:55.493371+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:01.194689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485444+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:18:55.485444+0000","last_peered":"2024-08-17T17:18:55.485444+0000","last_clean":"2024-08-17T17:18:55.485444+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:18:55.485444+0000","last_undegraded":"2024-08-17T17:18:55.485444+0000","last_fullsized":"2024-08-17T17:18:55.485444+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":474,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495530+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:18:55.495530+0000","last_peered":"2024-08-17T17:18:55.495530+0000","last_clean":"2024-08-17T17:18:55.495530+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:18:55.495530+0000","last_undegraded":"2024-08-17T17:18:55.495530+0000","last_fullsized":"2024-08-17T17:18:55.495530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489396+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:18:55.489396+0000","last_peered":"2024-08-17T17:18:55.489396+0000","last_clean":"2024-08-17T17:18:55.489396+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:18:55.489396+0000","last_undegraded":"2024-08-17T17:18:55.489396+0000","last_fullsized":"2024-08-17T17:18:55.489396+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487978+0000","last_change":"2024-08-17T17:17:50.473987+0000","last_active":"2024-08-17T17:18:55.487978+0000","last_peered":"2024-08-17T17:18:55.487978+0000","last_clean":"2024-08-17T17:18:55.487978+0000","last_became_active":"2024-08-17T17:17:50.472613+0000","last_became_peered":"2024-08-17T17:17:50.472613+0000","last_unstale":"2024-08-17T17:18:55.487978+0000","last_undegraded":"2024-08-17T17:18:55.487978+0000","last_fullsized":"2024-08-17T17:18:55.487978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:35.597393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":975,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489113+0000","last_change":"2024-08-17T17:17:50.473465+0000","last_active":"2024-08-17T17:18:55.489113+0000","last_peered":"2024-08-17T17:18:55.489113+0000","last_clean":"2024-08-17T17:18:55.489113+0000","last_became_active":"2024-08-17T17:17:50.472957+0000","last_became_peered":"2024-08-17T17:17:50.472957+0000","last_unstale":"2024-08-17T17:18:55.489113+0000","last_undegraded":"2024-08-17T17:18:55.489113+0000","last_fullsized":"2024-08-17T17:18:55.489113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:46.964083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1459,"num_read_kb":1465,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484270+0000","last_change":"2024-08-17T17:17:19.403089+0000","last_active":"2024-08-17T17:18:54.484270+0000","last_peered":"2024-08-17T17:18:54.484270+0000","last_clean":"2024-08-17T17:18:54.484270+0000","last_became_active":"2024-08-17T17:17:19.402747+0000","last_became_peered":"2024-08-17T17:17:19.402747+0000","last_unstale":"2024-08-17T17:18:54.484270+0000","last_undegraded":"2024-08-17T17:18:54.484270+0000","last_fullsized":"2024-08-17T17:18:54.484270+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:02.106769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496544+0000","last_change":"2024-08-17T17:18:55.496544+0000","last_active":"2024-08-17T17:18:55.496544+0000","last_peered":"2024-08-17T17:18:55.496544+0000","last_clean":"2024-08-17T17:18:55.496544+0000","last_became_active":"2024-08-17T17:18:55.495961+0000","last_became_peered":"2024-08-17T17:18:55.495961+0000","last_unstale":"2024-08-17T17:18:55.496544+0000","last_undegraded":"2024-08-17T17:18:55.496544+0000","last_fullsized":"2024-08-17T17:18:55.496544+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:43.998720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496478+0000","last_change":"2024-08-17T17:18:55.496478+0000","last_active":"2024-08-17T17:18:55.496478+0000","last_peered":"2024-08-17T17:18:55.496478+0000","last_clean":"2024-08-17T17:18:55.496478+0000","last_became_active":"2024-08-17T17:18:55.496095+0000","last_became_peered":"2024-08-17T17:18:55.496095+0000","last_unstale":"2024-08-17T17:18:55.496478+0000","last_undegraded":"2024-08-17T17:18:55.496478+0000","last_fullsized":"2024-08-17T17:18:55.496478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:46.676023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":359,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489793+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:18:55.489793+0000","last_peered":"2024-08-17T17:18:55.489793+0000","last_clean":"2024-08-17T17:18:55.489793+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:18:55.489793+0000","last_undegraded":"2024-08-17T17:18:55.489793+0000","last_fullsized":"2024-08-17T17:18:55.489793+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488698+0000","last_change":"2024-08-17T17:17:50.478401+0000","last_active":"2024-08-17T17:18:55.488698+0000","last_peered":"2024-08-17T17:18:55.488698+0000","last_clean":"2024-08-17T17:18:55.488698+0000","last_became_active":"2024-08-17T17:17:50.477120+0000","last_became_peered":"2024-08-17T17:17:50.477120+0000","last_unstale":"2024-08-17T17:18:55.488698+0000","last_undegraded":"2024-08-17T17:18:55.488698+0000","last_fullsized":"2024-08-17T17:18:55.488698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:21.133581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487869+0000","last_change":"2024-08-17T17:18:22.904542+0000","last_active":"2024-08-17T17:18:55.487869+0000","last_peered":"2024-08-17T17:18:55.487869+0000","last_clean":"2024-08-17T17:18:55.487869+0000","last_became_active":"2024-08-17T17:18:22.903998+0000","last_became_peered":"2024-08-17T17:18:22.903998+0000","last_unstale":"2024-08-17T17:18:55.487869+0000","last_undegraded":"2024-08-17T17:18:55.487869+0000","last_fullsized":"2024-08-17T17:18:55.487869+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:14.046789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":583,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487870+0000","last_change":"2024-08-17T17:17:19.409063+0000","last_active":"2024-08-17T17:18:55.487870+0000","last_peered":"2024-08-17T17:18:55.487870+0000","last_clean":"2024-08-17T17:18:55.487870+0000","last_became_active":"2024-08-17T17:17:19.408369+0000","last_became_peered":"2024-08-17T17:17:19.408369+0000","last_unstale":"2024-08-17T17:18:55.487870+0000","last_undegraded":"2024-08-17T17:18:55.487870+0000","last_fullsized":"2024-08-17T17:18:55.487870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:23.298610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484914+0000","last_change":"2024-08-17T17:17:19.406225+0000","last_active":"2024-08-17T17:18:55.484914+0000","last_peered":"2024-08-17T17:18:55.484914+0000","last_clean":"2024-08-17T17:18:55.484914+0000","last_became_active":"2024-08-17T17:17:19.405856+0000","last_became_peered":"2024-08-17T17:17:19.405856+0000","last_unstale":"2024-08-17T17:18:55.484914+0000","last_undegraded":"2024-08-17T17:18:55.484914+0000","last_fullsized":"2024-08-17T17:18:55.484914+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:09.665590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498816+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:18:55.498816+0000","last_peered":"2024-08-17T17:18:55.498816+0000","last_clean":"2024-08-17T17:18:55.498816+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:18:55.498816+0000","last_undegraded":"2024-08-17T17:18:55.498816+0000","last_fullsized":"2024-08-17T17:18:55.498816+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":523,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488035+0000","last_change":"2024-08-17T17:17:50.477255+0000","last_active":"2024-08-17T17:18:55.488035+0000","last_peered":"2024-08-17T17:18:55.488035+0000","last_clean":"2024-08-17T17:18:55.488035+0000","last_became_active":"2024-08-17T17:17:50.476780+0000","last_became_peered":"2024-08-17T17:17:50.476780+0000","last_unstale":"2024-08-17T17:18:55.488035+0000","last_undegraded":"2024-08-17T17:18:55.488035+0000","last_fullsized":"2024-08-17T17:18:55.488035+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:19.520906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497382+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:18:55.497382+0000","last_peered":"2024-08-17T17:18:55.497382+0000","last_clean":"2024-08-17T17:18:55.497382+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:18:55.497382+0000","last_undegraded":"2024-08-17T17:18:55.497382+0000","last_fullsized":"2024-08-17T17:18:55.497382+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484980+0000","last_change":"2024-08-17T17:17:50.478316+0000","last_active":"2024-08-17T17:18:55.484980+0000","last_peered":"2024-08-17T17:18:55.484980+0000","last_clean":"2024-08-17T17:18:55.484980+0000","last_became_active":"2024-08-17T17:17:50.477739+0000","last_became_peered":"2024-08-17T17:17:50.477739+0000","last_unstale":"2024-08-17T17:18:55.484980+0000","last_undegraded":"2024-08-17T17:18:55.484980+0000","last_fullsized":"2024-08-17T17:18:55.484980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:49.649407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":351,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493908+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:18:55.493908+0000","last_peered":"2024-08-17T17:18:55.493908+0000","last_clean":"2024-08-17T17:18:55.493908+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:18:55.493908+0000","last_undegraded":"2024-08-17T17:18:55.493908+0000","last_fullsized":"2024-08-17T17:18:55.493908+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482264+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:18:54.482264+0000","last_peered":"2024-08-17T17:18:54.482264+0000","last_clean":"2024-08-17T17:18:54.482264+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:18:54.482264+0000","last_undegraded":"2024-08-17T17:18:54.482264+0000","last_fullsized":"2024-08-17T17:18:54.482264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485723+0000","last_change":"2024-08-17T17:17:50.477504+0000","last_active":"2024-08-17T17:18:55.485723+0000","last_peered":"2024-08-17T17:18:55.485723+0000","last_clean":"2024-08-17T17:18:55.485723+0000","last_became_active":"2024-08-17T17:17:50.477172+0000","last_became_peered":"2024-08-17T17:17:50.477172+0000","last_unstale":"2024-08-17T17:18:55.485723+0000","last_undegraded":"2024-08-17T17:18:55.485723+0000","last_fullsized":"2024-08-17T17:18:55.485723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:19.990582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485166+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:18:55.485166+0000","last_peered":"2024-08-17T17:18:55.485166+0000","last_clean":"2024-08-17T17:18:55.485166+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:18:55.485166+0000","last_undegraded":"2024-08-17T17:18:55.485166+0000","last_fullsized":"2024-08-17T17:18:55.485166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488468+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:18:55.488468+0000","last_peered":"2024-08-17T17:18:55.488468+0000","last_clean":"2024-08-17T17:18:55.488468+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:18:55.488468+0000","last_undegraded":"2024-08-17T17:18:55.488468+0000","last_fullsized":"2024-08-17T17:18:55.488468+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495009+0000","last_change":"2024-08-17T17:18:55.495009+0000","last_active":"2024-08-17T17:18:55.495009+0000","last_peered":"2024-08-17T17:18:55.495009+0000","last_clean":"2024-08-17T17:18:55.495009+0000","last_became_active":"2024-08-17T17:18:55.494532+0000","last_became_peered":"2024-08-17T17:18:55.494532+0000","last_unstale":"2024-08-17T17:18:55.495009+0000","last_undegraded":"2024-08-17T17:18:55.495009+0000","last_fullsized":"2024-08-17T17:18:55.495009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:54.695395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489852+0000","last_change":"2024-08-17T17:17:50.474706+0000","last_active":"2024-08-17T17:18:54.489852+0000","last_peered":"2024-08-17T17:18:54.489852+0000","last_clean":"2024-08-17T17:18:54.489852+0000","last_became_active":"2024-08-17T17:17:50.474392+0000","last_became_peered":"2024-08-17T17:17:50.474392+0000","last_unstale":"2024-08-17T17:18:54.489852+0000","last_undegraded":"2024-08-17T17:18:54.489852+0000","last_fullsized":"2024-08-17T17:18:54.489852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:38.744232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498410+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:18:55.498410+0000","last_peered":"2024-08-17T17:18:55.498410+0000","last_clean":"2024-08-17T17:18:55.498410+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:18:55.498410+0000","last_undegraded":"2024-08-17T17:18:55.498410+0000","last_fullsized":"2024-08-17T17:18:55.498410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494317+0000","last_change":"2024-08-17T17:18:55.493741+0000","last_active":"2024-08-17T17:18:55.494317+0000","last_peered":"2024-08-17T17:18:55.494317+0000","last_clean":"2024-08-17T17:18:55.494317+0000","last_became_active":"2024-08-17T17:18:55.493423+0000","last_became_peered":"2024-08-17T17:18:55.493423+0000","last_unstale":"2024-08-17T17:18:55.494317+0000","last_undegraded":"2024-08-17T17:18:55.494317+0000","last_fullsized":"2024-08-17T17:18:55.494317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:15.411411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497350+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:18:55.497350+0000","last_peered":"2024-08-17T17:18:55.497350+0000","last_clean":"2024-08-17T17:18:55.497350+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:18:55.497350+0000","last_undegraded":"2024-08-17T17:18:55.497350+0000","last_fullsized":"2024-08-17T17:18:55.497350+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487419+0000","last_change":"2024-08-17T17:17:50.474242+0000","last_active":"2024-08-17T17:18:54.487419+0000","last_peered":"2024-08-17T17:18:54.487419+0000","last_clean":"2024-08-17T17:18:54.487419+0000","last_became_active":"2024-08-17T17:17:50.473923+0000","last_became_peered":"2024-08-17T17:17:50.473923+0000","last_unstale":"2024-08-17T17:18:54.487419+0000","last_undegraded":"2024-08-17T17:18:54.487419+0000","last_fullsized":"2024-08-17T17:18:54.487419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:13.899933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493594+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:18:55.493594+0000","last_peered":"2024-08-17T17:18:55.493594+0000","last_clean":"2024-08-17T17:18:55.493594+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:18:55.493594+0000","last_undegraded":"2024-08-17T17:18:55.493594+0000","last_fullsized":"2024-08-17T17:18:55.493594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499603+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:18:55.499603+0000","last_peered":"2024-08-17T17:18:55.499603+0000","last_clean":"2024-08-17T17:18:55.499603+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:18:55.499603+0000","last_undegraded":"2024-08-17T17:18:55.499603+0000","last_fullsized":"2024-08-17T17:18:55.499603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495398+0000","last_change":"2024-08-17T17:18:55.495340+0000","last_active":"2024-08-17T17:18:55.495398+0000","last_peered":"2024-08-17T17:18:55.495398+0000","last_clean":"2024-08-17T17:18:55.495398+0000","last_became_active":"2024-08-17T17:18:55.495057+0000","last_became_peered":"2024-08-17T17:18:55.495057+0000","last_unstale":"2024-08-17T17:18:55.495398+0000","last_undegraded":"2024-08-17T17:18:55.495398+0000","last_fullsized":"2024-08-17T17:18:55.495398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:26.993423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498393+0000","last_change":"2024-08-17T17:18:55.498330+0000","last_active":"2024-08-17T17:18:55.498393+0000","last_peered":"2024-08-17T17:18:55.498393+0000","last_clean":"2024-08-17T17:18:55.498393+0000","last_became_active":"2024-08-17T17:18:55.496964+0000","last_became_peered":"2024-08-17T17:18:55.496964+0000","last_unstale":"2024-08-17T17:18:55.498393+0000","last_undegraded":"2024-08-17T17:18:55.498393+0000","last_fullsized":"2024-08-17T17:18:55.498393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:09:56.482828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482472+0000","last_change":"2024-08-17T17:18:22.904972+0000","last_active":"2024-08-17T17:18:54.482472+0000","last_peered":"2024-08-17T17:18:54.482472+0000","last_clean":"2024-08-17T17:18:54.482472+0000","last_became_active":"2024-08-17T17:18:22.904630+0000","last_became_peered":"2024-08-17T17:18:22.904630+0000","last_unstale":"2024-08-17T17:18:54.482472+0000","last_undegraded":"2024-08-17T17:18:54.482472+0000","last_fullsized":"2024-08-17T17:18:54.482472+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:01.926349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494019+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:18:55.494019+0000","last_peered":"2024-08-17T17:18:55.494019+0000","last_clean":"2024-08-17T17:18:55.494019+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:18:55.494019+0000","last_undegraded":"2024-08-17T17:18:55.494019+0000","last_fullsized":"2024-08-17T17:18:55.494019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484803+0000","last_change":"2024-08-17T17:17:19.406837+0000","last_active":"2024-08-17T17:18:55.484803+0000","last_peered":"2024-08-17T17:18:55.484803+0000","last_clean":"2024-08-17T17:18:55.484803+0000","last_became_active":"2024-08-17T17:17:19.406413+0000","last_became_peered":"2024-08-17T17:17:19.406413+0000","last_unstale":"2024-08-17T17:18:55.484803+0000","last_undegraded":"2024-08-17T17:18:55.484803+0000","last_fullsized":"2024-08-17T17:18:55.484803+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:22.497997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498573+0000","last_change":"2024-08-17T17:18:55.498573+0000","last_active":"2024-08-17T17:18:55.498573+0000","last_peered":"2024-08-17T17:18:55.498573+0000","last_clean":"2024-08-17T17:18:55.498573+0000","last_became_active":"2024-08-17T17:18:55.498230+0000","last_became_peered":"2024-08-17T17:18:55.498230+0000","last_unstale":"2024-08-17T17:18:55.498573+0000","last_undegraded":"2024-08-17T17:18:55.498573+0000","last_fullsized":"2024-08-17T17:18:55.498573+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.161705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.491060+0000","last_change":"2024-08-17T17:17:50.472168+0000","last_active":"2024-08-17T17:18:54.491060+0000","last_peered":"2024-08-17T17:18:54.491060+0000","last_clean":"2024-08-17T17:18:54.491060+0000","last_became_active":"2024-08-17T17:17:50.471864+0000","last_became_peered":"2024-08-17T17:17:50.471864+0000","last_unstale":"2024-08-17T17:18:54.491060+0000","last_undegraded":"2024-08-17T17:18:54.491060+0000","last_fullsized":"2024-08-17T17:18:54.491060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:32.866769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492171+0000","last_change":"2024-08-17T17:18:55.492171+0000","last_active":"2024-08-17T17:18:55.492171+0000","last_peered":"2024-08-17T17:18:55.492171+0000","last_clean":"2024-08-17T17:18:55.492171+0000","last_became_active":"2024-08-17T17:18:55.491790+0000","last_became_peered":"2024-08-17T17:18:55.491790+0000","last_unstale":"2024-08-17T17:18:55.492171+0000","last_undegraded":"2024-08-17T17:18:55.492171+0000","last_fullsized":"2024-08-17T17:18:55.492171+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:44:21.883054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.490901+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:18:54.490901+0000","last_peered":"2024-08-17T17:18:54.490901+0000","last_clean":"2024-08-17T17:18:54.490901+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:18:54.490901+0000","last_undegraded":"2024-08-17T17:18:54.490901+0000","last_fullsized":"2024-08-17T17:18:54.490901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.500485+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:18:55.500485+0000","last_peered":"2024-08-17T17:18:55.500485+0000","last_clean":"2024-08-17T17:18:55.500485+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:18:55.500485+0000","last_undegraded":"2024-08-17T17:18:55.500485+0000","last_fullsized":"2024-08-17T17:18:55.500485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498299+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:18:55.498299+0000","last_peered":"2024-08-17T17:18:55.498299+0000","last_clean":"2024-08-17T17:18:55.498299+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:18:55.498299+0000","last_undegraded":"2024-08-17T17:18:55.498299+0000","last_fullsized":"2024-08-17T17:18:55.498299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482239+0000","last_change":"2024-08-17T17:18:22.903634+0000","last_active":"2024-08-17T17:18:54.482239+0000","last_peered":"2024-08-17T17:18:54.482239+0000","last_clean":"2024-08-17T17:18:54.482239+0000","last_became_active":"2024-08-17T17:18:22.903279+0000","last_became_peered":"2024-08-17T17:18:22.903279+0000","last_unstale":"2024-08-17T17:18:54.482239+0000","last_undegraded":"2024-08-17T17:18:54.482239+0000","last_fullsized":"2024-08-17T17:18:54.482239+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:08.309006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.483800+0000","last_change":"2024-08-17T17:18:22.902105+0000","last_active":"2024-08-17T17:18:55.483800+0000","last_peered":"2024-08-17T17:18:55.483800+0000","last_clean":"2024-08-17T17:18:55.483800+0000","last_became_active":"2024-08-17T17:18:22.901714+0000","last_became_peered":"2024-08-17T17:18:22.901714+0000","last_unstale":"2024-08-17T17:18:55.483800+0000","last_undegraded":"2024-08-17T17:18:55.483800+0000","last_fullsized":"2024-08-17T17:18:55.483800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:33.515070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201477,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36996,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708156,"statfs":{"total":95995035648,"available":95957151744,"internally_reserved":0,"allocated":6021120,"data_stored":5413201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463115,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1772,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1814528,"data_stored":1218505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724751,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32308,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712844,"statfs":{"total":95995035648,"available":95961952256,"internally_reserved":0,"allocated":1216512,"data_stored":624228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986387,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":5296,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":5423104,"data_stored":4822771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1180,"kb_used_omap":5,"kb_used_meta":27514,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1208320,"data_stored":623667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":28174409},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598805,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1212416,"data_stored":624193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":28172461},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1176,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1204224,"data_stored":623323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":28172705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925858,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6004736,"data_stored":5409126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":28172892},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:20:01.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:20:01.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:01 smithi064 ceph-mon[104516]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:01.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:01 smithi064 ceph-mon[109246]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:01 smithi175 ceph-mon[101434]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:02.725 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:02.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:02 smithi064 ceph-mon[104516]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:02.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:02 smithi064 ceph-mon[109246]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:02 smithi175 ceph-mon[101434]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:04.247 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:03] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-08-17T17:20:04.447 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:20:04.448 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:20:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:04 smithi064 ceph-mon[104516]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:04.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:04 smithi064 ceph-mon[109246]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:05.077 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2024-08-17T17:20:02.696989+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":254080,"kb_used_data":23540,"kb_used_omap":34,"kb_used_meta":230365,"kb_avail":749707136,"statfs":{"total":767960285184,"available":767700107264,"internally_reserved":0,"allocated":24104960,"data_stored":19359014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35823,"internal_metadata":235893777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002586"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499047+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:18:55.499047+0000","last_peered":"2024-08-17T17:18:55.499047+0000","last_clean":"2024-08-17T17:18:55.499047+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:18:55.499047+0000","last_undegraded":"2024-08-17T17:18:55.499047+0000","last_fullsized":"2024-08-17T17:18:55.499047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487943+0000","last_change":"2024-08-17T17:18:22.909422+0000","last_active":"2024-08-17T17:18:55.487943+0000","last_peered":"2024-08-17T17:18:55.487943+0000","last_clean":"2024-08-17T17:18:55.487943+0000","last_became_active":"2024-08-17T17:18:22.909017+0000","last_became_peered":"2024-08-17T17:18:22.909017+0000","last_unstale":"2024-08-17T17:18:55.487943+0000","last_undegraded":"2024-08-17T17:18:55.487943+0000","last_fullsized":"2024-08-17T17:18:55.487943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:31.910796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495867+0000","last_change":"2024-08-17T17:18:55.495867+0000","last_active":"2024-08-17T17:18:55.495867+0000","last_peered":"2024-08-17T17:18:55.495867+0000","last_clean":"2024-08-17T17:18:55.495867+0000","last_became_active":"2024-08-17T17:18:55.495425+0000","last_became_peered":"2024-08-17T17:18:55.495425+0000","last_unstale":"2024-08-17T17:18:55.495867+0000","last_undegraded":"2024-08-17T17:18:55.495867+0000","last_fullsized":"2024-08-17T17:18:55.495867+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:35.743221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494262+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:18:55.494262+0000","last_peered":"2024-08-17T17:18:55.494262+0000","last_clean":"2024-08-17T17:18:55.494262+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:18:55.494262+0000","last_undegraded":"2024-08-17T17:18:55.494262+0000","last_fullsized":"2024-08-17T17:18:55.494262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497035+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:18:55.497035+0000","last_peered":"2024-08-17T17:18:55.497035+0000","last_clean":"2024-08-17T17:18:55.497035+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:18:55.497035+0000","last_undegraded":"2024-08-17T17:18:55.497035+0000","last_fullsized":"2024-08-17T17:18:55.497035+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488108+0000","last_change":"2024-08-17T17:05:34.074405+0000","last_active":"2024-08-17T17:18:55.488108+0000","last_peered":"2024-08-17T17:18:55.488108+0000","last_clean":"2024-08-17T17:18:55.488108+0000","last_became_active":"2024-08-17T17:05:34.073920+0000","last_became_peered":"2024-08-17T17:05:34.073920+0000","last_unstale":"2024-08-17T17:18:55.488108+0000","last_undegraded":"2024-08-17T17:18:55.488108+0000","last_fullsized":"2024-08-17T17:18:55.488108+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:57.682428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495715+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:18:55.495715+0000","last_peered":"2024-08-17T17:18:55.495715+0000","last_clean":"2024-08-17T17:18:55.495715+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:18:55.495715+0000","last_undegraded":"2024-08-17T17:18:55.495715+0000","last_fullsized":"2024-08-17T17:18:55.495715+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493569+0000","last_change":"2024-08-17T17:17:19.404513+0000","last_active":"2024-08-17T17:18:55.493569+0000","last_peered":"2024-08-17T17:18:55.493569+0000","last_clean":"2024-08-17T17:18:55.493569+0000","last_became_active":"2024-08-17T17:17:19.404148+0000","last_became_peered":"2024-08-17T17:17:19.404148+0000","last_unstale":"2024-08-17T17:18:55.493569+0000","last_undegraded":"2024-08-17T17:18:55.493569+0000","last_fullsized":"2024-08-17T17:18:55.493569+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:10.729553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487976+0000","last_change":"2024-08-17T17:17:19.408816+0000","last_active":"2024-08-17T17:18:55.487976+0000","last_peered":"2024-08-17T17:18:55.487976+0000","last_clean":"2024-08-17T17:18:55.487976+0000","last_became_active":"2024-08-17T17:17:19.408449+0000","last_became_peered":"2024-08-17T17:17:19.408449+0000","last_unstale":"2024-08-17T17:18:55.487976+0000","last_undegraded":"2024-08-17T17:18:55.487976+0000","last_fullsized":"2024-08-17T17:18:55.487976+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:20.970954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489725+0000","last_change":"2024-08-17T17:05:34.259360+0000","last_active":"2024-08-17T17:18:54.489725+0000","last_peered":"2024-08-17T17:18:54.489725+0000","last_clean":"2024-08-17T17:18:54.489725+0000","last_became_active":"2024-08-17T17:05:34.259074+0000","last_became_peered":"2024-08-17T17:05:34.259074+0000","last_unstale":"2024-08-17T17:18:54.489725+0000","last_undegraded":"2024-08-17T17:18:54.489725+0000","last_fullsized":"2024-08-17T17:18:54.489725+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:58.820200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487651+0000","last_change":"2024-08-17T17:17:50.478014+0000","last_active":"2024-08-17T17:18:55.487651+0000","last_peered":"2024-08-17T17:18:55.487651+0000","last_clean":"2024-08-17T17:18:55.487651+0000","last_became_active":"2024-08-17T17:17:50.477662+0000","last_became_peered":"2024-08-17T17:17:50.477662+0000","last_unstale":"2024-08-17T17:18:55.487651+0000","last_undegraded":"2024-08-17T17:18:55.487651+0000","last_fullsized":"2024-08-17T17:18:55.487651+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:46.006989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498099+0000","last_change":"2024-08-17T17:18:55.498099+0000","last_active":"2024-08-17T17:18:55.498099+0000","last_peered":"2024-08-17T17:18:55.498099+0000","last_clean":"2024-08-17T17:18:55.498099+0000","last_became_active":"2024-08-17T17:18:55.497722+0000","last_became_peered":"2024-08-17T17:18:55.497722+0000","last_unstale":"2024-08-17T17:18:55.498099+0000","last_undegraded":"2024-08-17T17:18:55.498099+0000","last_fullsized":"2024-08-17T17:18:55.498099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:49.290137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487769+0000","last_change":"2024-08-17T17:05:34.068845+0000","last_active":"2024-08-17T17:18:55.487769+0000","last_peered":"2024-08-17T17:18:55.487769+0000","last_clean":"2024-08-17T17:18:55.487769+0000","last_became_active":"2024-08-17T17:05:34.068416+0000","last_became_peered":"2024-08-17T17:05:34.068416+0000","last_unstale":"2024-08-17T17:18:55.487769+0000","last_undegraded":"2024-08-17T17:18:55.487769+0000","last_fullsized":"2024-08-17T17:18:55.487769+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:40.483524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486337+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:18:54.486337+0000","last_peered":"2024-08-17T17:18:54.486337+0000","last_clean":"2024-08-17T17:18:54.486337+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:18:54.486337+0000","last_undegraded":"2024-08-17T17:18:54.486337+0000","last_fullsized":"2024-08-17T17:18:54.486337+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497126+0000","last_change":"2024-08-17T17:18:55.496653+0000","last_active":"2024-08-17T17:18:55.497126+0000","last_peered":"2024-08-17T17:18:55.497126+0000","last_clean":"2024-08-17T17:18:55.497126+0000","last_became_active":"2024-08-17T17:18:55.495980+0000","last_became_peered":"2024-08-17T17:18:55.495980+0000","last_unstale":"2024-08-17T17:18:55.497126+0000","last_undegraded":"2024-08-17T17:18:55.497126+0000","last_fullsized":"2024-08-17T17:18:55.497126+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:34.698405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484986+0000","last_change":"2024-08-17T17:17:50.472859+0000","last_active":"2024-08-17T17:18:55.484986+0000","last_peered":"2024-08-17T17:18:55.484986+0000","last_clean":"2024-08-17T17:18:55.484986+0000","last_became_active":"2024-08-17T17:17:50.472564+0000","last_became_peered":"2024-08-17T17:17:50.472564+0000","last_unstale":"2024-08-17T17:18:55.484986+0000","last_undegraded":"2024-08-17T17:18:55.484986+0000","last_fullsized":"2024-08-17T17:18:55.484986+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:25.759541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499163+0000","last_change":"2024-08-17T17:18:55.499100+0000","last_active":"2024-08-17T17:18:55.499163+0000","last_peered":"2024-08-17T17:18:55.499163+0000","last_clean":"2024-08-17T17:18:55.499163+0000","last_became_active":"2024-08-17T17:18:55.498395+0000","last_became_peered":"2024-08-17T17:18:55.498395+0000","last_unstale":"2024-08-17T17:18:55.499163+0000","last_undegraded":"2024-08-17T17:18:55.499163+0000","last_fullsized":"2024-08-17T17:18:55.499163+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:19.200301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498811+0000","last_change":"2024-08-17T17:18:55.498039+0000","last_active":"2024-08-17T17:18:55.498811+0000","last_peered":"2024-08-17T17:18:55.498811+0000","last_clean":"2024-08-17T17:18:55.498811+0000","last_became_active":"2024-08-17T17:18:55.497685+0000","last_became_peered":"2024-08-17T17:18:55.497685+0000","last_unstale":"2024-08-17T17:18:55.498811+0000","last_undegraded":"2024-08-17T17:18:55.498811+0000","last_fullsized":"2024-08-17T17:18:55.498811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:49.171022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486783+0000","last_change":"2024-08-17T17:17:50.479775+0000","last_active":"2024-08-17T17:18:54.486783+0000","last_peered":"2024-08-17T17:18:54.486783+0000","last_clean":"2024-08-17T17:18:54.486783+0000","last_became_active":"2024-08-17T17:17:50.479361+0000","last_became_peered":"2024-08-17T17:17:50.479361+0000","last_unstale":"2024-08-17T17:18:54.486783+0000","last_undegraded":"2024-08-17T17:18:54.486783+0000","last_fullsized":"2024-08-17T17:18:54.486783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:52:13.653931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485498+0000","last_change":"2024-08-17T17:18:22.904844+0000","last_active":"2024-08-17T17:18:55.485498+0000","last_peered":"2024-08-17T17:18:55.485498+0000","last_clean":"2024-08-17T17:18:55.485498+0000","last_became_active":"2024-08-17T17:18:22.904512+0000","last_became_peered":"2024-08-17T17:18:22.904512+0000","last_unstale":"2024-08-17T17:18:55.485498+0000","last_undegraded":"2024-08-17T17:18:55.485498+0000","last_fullsized":"2024-08-17T17:18:55.485498+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:35.381256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484072+0000","last_change":"2024-08-17T17:18:22.904885+0000","last_active":"2024-08-17T17:18:54.484072+0000","last_peered":"2024-08-17T17:18:54.484072+0000","last_clean":"2024-08-17T17:18:54.484072+0000","last_became_active":"2024-08-17T17:18:22.904535+0000","last_became_peered":"2024-08-17T17:18:22.904535+0000","last_unstale":"2024-08-17T17:18:54.484072+0000","last_undegraded":"2024-08-17T17:18:54.484072+0000","last_fullsized":"2024-08-17T17:18:54.484072+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:00.547235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485431+0000","last_change":"2024-08-17T17:17:50.477509+0000","last_active":"2024-08-17T17:18:55.485431+0000","last_peered":"2024-08-17T17:18:55.485431+0000","last_clean":"2024-08-17T17:18:55.485431+0000","last_became_active":"2024-08-17T17:17:50.477204+0000","last_became_peered":"2024-08-17T17:17:50.477204+0000","last_unstale":"2024-08-17T17:18:55.485431+0000","last_undegraded":"2024-08-17T17:18:55.485431+0000","last_fullsized":"2024-08-17T17:18:55.485431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:02.378683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498012+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:18:55.498012+0000","last_peered":"2024-08-17T17:18:55.498012+0000","last_clean":"2024-08-17T17:18:55.498012+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:18:55.498012+0000","last_undegraded":"2024-08-17T17:18:55.498012+0000","last_fullsized":"2024-08-17T17:18:55.498012+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492775+0000","last_change":"2024-08-17T17:18:55.492713+0000","last_active":"2024-08-17T17:18:55.492775+0000","last_peered":"2024-08-17T17:18:55.492775+0000","last_clean":"2024-08-17T17:18:55.492775+0000","last_became_active":"2024-08-17T17:18:55.492385+0000","last_became_peered":"2024-08-17T17:18:55.492385+0000","last_unstale":"2024-08-17T17:18:55.492775+0000","last_undegraded":"2024-08-17T17:18:55.492775+0000","last_fullsized":"2024-08-17T17:18:55.492775+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:33.301910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":402,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487488+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:18:55.487488+0000","last_peered":"2024-08-17T17:18:55.487488+0000","last_clean":"2024-08-17T17:18:55.487488+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:18:55.487488+0000","last_undegraded":"2024-08-17T17:18:55.487488+0000","last_fullsized":"2024-08-17T17:18:55.487488+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489011+0000","last_change":"2024-08-17T17:18:22.902347+0000","last_active":"2024-08-17T17:18:55.489011+0000","last_peered":"2024-08-17T17:18:55.489011+0000","last_clean":"2024-08-17T17:18:55.489011+0000","last_became_active":"2024-08-17T17:18:22.901676+0000","last_became_peered":"2024-08-17T17:18:22.901676+0000","last_unstale":"2024-08-17T17:18:55.489011+0000","last_undegraded":"2024-08-17T17:18:55.489011+0000","last_fullsized":"2024-08-17T17:18:55.489011+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:45:20.310108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":829,"num_read_kb":883,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499720+0000","last_change":"2024-08-17T17:18:55.499664+0000","last_active":"2024-08-17T17:18:55.499720+0000","last_peered":"2024-08-17T17:18:55.499720+0000","last_clean":"2024-08-17T17:18:55.499720+0000","last_became_active":"2024-08-17T17:18:55.499216+0000","last_became_peered":"2024-08-17T17:18:55.499216+0000","last_unstale":"2024-08-17T17:18:55.499720+0000","last_undegraded":"2024-08-17T17:18:55.499720+0000","last_fullsized":"2024-08-17T17:18:55.499720+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:38.025283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487844+0000","last_change":"2024-08-17T17:05:34.261058+0000","last_active":"2024-08-17T17:18:54.487844+0000","last_peered":"2024-08-17T17:18:54.487844+0000","last_clean":"2024-08-17T17:18:54.487844+0000","last_became_active":"2024-08-17T17:05:34.260567+0000","last_became_peered":"2024-08-17T17:05:34.260567+0000","last_unstale":"2024-08-17T17:18:54.487844+0000","last_undegraded":"2024-08-17T17:18:54.487844+0000","last_fullsized":"2024-08-17T17:18:54.487844+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:06.943295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":479,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.486454+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:18:54.486454+0000","last_peered":"2024-08-17T17:18:54.486454+0000","last_clean":"2024-08-17T17:18:54.486454+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:18:54.486454+0000","last_undegraded":"2024-08-17T17:18:54.486454+0000","last_fullsized":"2024-08-17T17:18:54.486454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493371+0000","last_change":"2024-08-17T17:18:55.492730+0000","last_active":"2024-08-17T17:18:55.493371+0000","last_peered":"2024-08-17T17:18:55.493371+0000","last_clean":"2024-08-17T17:18:55.493371+0000","last_became_active":"2024-08-17T17:18:55.492439+0000","last_became_peered":"2024-08-17T17:18:55.492439+0000","last_unstale":"2024-08-17T17:18:55.493371+0000","last_undegraded":"2024-08-17T17:18:55.493371+0000","last_fullsized":"2024-08-17T17:18:55.493371+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:01.194689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485444+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:18:55.485444+0000","last_peered":"2024-08-17T17:18:55.485444+0000","last_clean":"2024-08-17T17:18:55.485444+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:18:55.485444+0000","last_undegraded":"2024-08-17T17:18:55.485444+0000","last_fullsized":"2024-08-17T17:18:55.485444+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":474,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495530+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:18:55.495530+0000","last_peered":"2024-08-17T17:18:55.495530+0000","last_clean":"2024-08-17T17:18:55.495530+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:18:55.495530+0000","last_undegraded":"2024-08-17T17:18:55.495530+0000","last_fullsized":"2024-08-17T17:18:55.495530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489396+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:18:55.489396+0000","last_peered":"2024-08-17T17:18:55.489396+0000","last_clean":"2024-08-17T17:18:55.489396+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:18:55.489396+0000","last_undegraded":"2024-08-17T17:18:55.489396+0000","last_fullsized":"2024-08-17T17:18:55.489396+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487978+0000","last_change":"2024-08-17T17:17:50.473987+0000","last_active":"2024-08-17T17:18:55.487978+0000","last_peered":"2024-08-17T17:18:55.487978+0000","last_clean":"2024-08-17T17:18:55.487978+0000","last_became_active":"2024-08-17T17:17:50.472613+0000","last_became_peered":"2024-08-17T17:17:50.472613+0000","last_unstale":"2024-08-17T17:18:55.487978+0000","last_undegraded":"2024-08-17T17:18:55.487978+0000","last_fullsized":"2024-08-17T17:18:55.487978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:35.597393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":975,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489113+0000","last_change":"2024-08-17T17:17:50.473465+0000","last_active":"2024-08-17T17:18:55.489113+0000","last_peered":"2024-08-17T17:18:55.489113+0000","last_clean":"2024-08-17T17:18:55.489113+0000","last_became_active":"2024-08-17T17:17:50.472957+0000","last_became_peered":"2024-08-17T17:17:50.472957+0000","last_unstale":"2024-08-17T17:18:55.489113+0000","last_undegraded":"2024-08-17T17:18:55.489113+0000","last_fullsized":"2024-08-17T17:18:55.489113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:46.964083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1459,"num_read_kb":1465,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.484270+0000","last_change":"2024-08-17T17:17:19.403089+0000","last_active":"2024-08-17T17:18:54.484270+0000","last_peered":"2024-08-17T17:18:54.484270+0000","last_clean":"2024-08-17T17:18:54.484270+0000","last_became_active":"2024-08-17T17:17:19.402747+0000","last_became_peered":"2024-08-17T17:17:19.402747+0000","last_unstale":"2024-08-17T17:18:54.484270+0000","last_undegraded":"2024-08-17T17:18:54.484270+0000","last_fullsized":"2024-08-17T17:18:54.484270+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:02.106769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496544+0000","last_change":"2024-08-17T17:18:55.496544+0000","last_active":"2024-08-17T17:18:55.496544+0000","last_peered":"2024-08-17T17:18:55.496544+0000","last_clean":"2024-08-17T17:18:55.496544+0000","last_became_active":"2024-08-17T17:18:55.495961+0000","last_became_peered":"2024-08-17T17:18:55.495961+0000","last_unstale":"2024-08-17T17:18:55.496544+0000","last_undegraded":"2024-08-17T17:18:55.496544+0000","last_fullsized":"2024-08-17T17:18:55.496544+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:43.998720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.496478+0000","last_change":"2024-08-17T17:18:55.496478+0000","last_active":"2024-08-17T17:18:55.496478+0000","last_peered":"2024-08-17T17:18:55.496478+0000","last_clean":"2024-08-17T17:18:55.496478+0000","last_became_active":"2024-08-17T17:18:55.496095+0000","last_became_peered":"2024-08-17T17:18:55.496095+0000","last_unstale":"2024-08-17T17:18:55.496478+0000","last_undegraded":"2024-08-17T17:18:55.496478+0000","last_fullsized":"2024-08-17T17:18:55.496478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:46.676023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":359,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.489793+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:18:55.489793+0000","last_peered":"2024-08-17T17:18:55.489793+0000","last_clean":"2024-08-17T17:18:55.489793+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:18:55.489793+0000","last_undegraded":"2024-08-17T17:18:55.489793+0000","last_fullsized":"2024-08-17T17:18:55.489793+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488698+0000","last_change":"2024-08-17T17:17:50.478401+0000","last_active":"2024-08-17T17:18:55.488698+0000","last_peered":"2024-08-17T17:18:55.488698+0000","last_clean":"2024-08-17T17:18:55.488698+0000","last_became_active":"2024-08-17T17:17:50.477120+0000","last_became_peered":"2024-08-17T17:17:50.477120+0000","last_unstale":"2024-08-17T17:18:55.488698+0000","last_undegraded":"2024-08-17T17:18:55.488698+0000","last_fullsized":"2024-08-17T17:18:55.488698+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:21.133581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487869+0000","last_change":"2024-08-17T17:18:22.904542+0000","last_active":"2024-08-17T17:18:55.487869+0000","last_peered":"2024-08-17T17:18:55.487869+0000","last_clean":"2024-08-17T17:18:55.487869+0000","last_became_active":"2024-08-17T17:18:22.903998+0000","last_became_peered":"2024-08-17T17:18:22.903998+0000","last_unstale":"2024-08-17T17:18:55.487869+0000","last_undegraded":"2024-08-17T17:18:55.487869+0000","last_fullsized":"2024-08-17T17:18:55.487869+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:14.046789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":583,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.487870+0000","last_change":"2024-08-17T17:17:19.409063+0000","last_active":"2024-08-17T17:18:55.487870+0000","last_peered":"2024-08-17T17:18:55.487870+0000","last_clean":"2024-08-17T17:18:55.487870+0000","last_became_active":"2024-08-17T17:17:19.408369+0000","last_became_peered":"2024-08-17T17:17:19.408369+0000","last_unstale":"2024-08-17T17:18:55.487870+0000","last_undegraded":"2024-08-17T17:18:55.487870+0000","last_fullsized":"2024-08-17T17:18:55.487870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:23.298610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484914+0000","last_change":"2024-08-17T17:17:19.406225+0000","last_active":"2024-08-17T17:18:55.484914+0000","last_peered":"2024-08-17T17:18:55.484914+0000","last_clean":"2024-08-17T17:18:55.484914+0000","last_became_active":"2024-08-17T17:17:19.405856+0000","last_became_peered":"2024-08-17T17:17:19.405856+0000","last_unstale":"2024-08-17T17:18:55.484914+0000","last_undegraded":"2024-08-17T17:18:55.484914+0000","last_fullsized":"2024-08-17T17:18:55.484914+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:09.665590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498816+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:18:55.498816+0000","last_peered":"2024-08-17T17:18:55.498816+0000","last_clean":"2024-08-17T17:18:55.498816+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:18:55.498816+0000","last_undegraded":"2024-08-17T17:18:55.498816+0000","last_fullsized":"2024-08-17T17:18:55.498816+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":523,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488035+0000","last_change":"2024-08-17T17:17:50.477255+0000","last_active":"2024-08-17T17:18:55.488035+0000","last_peered":"2024-08-17T17:18:55.488035+0000","last_clean":"2024-08-17T17:18:55.488035+0000","last_became_active":"2024-08-17T17:17:50.476780+0000","last_became_peered":"2024-08-17T17:17:50.476780+0000","last_unstale":"2024-08-17T17:18:55.488035+0000","last_undegraded":"2024-08-17T17:18:55.488035+0000","last_fullsized":"2024-08-17T17:18:55.488035+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:19.520906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497382+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:18:55.497382+0000","last_peered":"2024-08-17T17:18:55.497382+0000","last_clean":"2024-08-17T17:18:55.497382+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:18:55.497382+0000","last_undegraded":"2024-08-17T17:18:55.497382+0000","last_fullsized":"2024-08-17T17:18:55.497382+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484980+0000","last_change":"2024-08-17T17:17:50.478316+0000","last_active":"2024-08-17T17:18:55.484980+0000","last_peered":"2024-08-17T17:18:55.484980+0000","last_clean":"2024-08-17T17:18:55.484980+0000","last_became_active":"2024-08-17T17:17:50.477739+0000","last_became_peered":"2024-08-17T17:17:50.477739+0000","last_unstale":"2024-08-17T17:18:55.484980+0000","last_undegraded":"2024-08-17T17:18:55.484980+0000","last_fullsized":"2024-08-17T17:18:55.484980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:49.649407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":351,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493908+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:18:55.493908+0000","last_peered":"2024-08-17T17:18:55.493908+0000","last_clean":"2024-08-17T17:18:55.493908+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:18:55.493908+0000","last_undegraded":"2024-08-17T17:18:55.493908+0000","last_fullsized":"2024-08-17T17:18:55.493908+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482264+0000","last_change":"2024-08-17T17:05:34.093029+0000","last_active":"2024-08-17T17:18:54.482264+0000","last_peered":"2024-08-17T17:18:54.482264+0000","last_clean":"2024-08-17T17:18:54.482264+0000","last_became_active":"2024-08-17T17:05:34.092722+0000","last_became_peered":"2024-08-17T17:05:34.092722+0000","last_unstale":"2024-08-17T17:18:54.482264+0000","last_undegraded":"2024-08-17T17:18:54.482264+0000","last_fullsized":"2024-08-17T17:18:54.482264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:34.899559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485723+0000","last_change":"2024-08-17T17:17:50.477504+0000","last_active":"2024-08-17T17:18:55.485723+0000","last_peered":"2024-08-17T17:18:55.485723+0000","last_clean":"2024-08-17T17:18:55.485723+0000","last_became_active":"2024-08-17T17:17:50.477172+0000","last_became_peered":"2024-08-17T17:17:50.477172+0000","last_unstale":"2024-08-17T17:18:55.485723+0000","last_undegraded":"2024-08-17T17:18:55.485723+0000","last_fullsized":"2024-08-17T17:18:55.485723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:19.990582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.485166+0000","last_change":"2024-08-17T17:05:34.072897+0000","last_active":"2024-08-17T17:18:55.485166+0000","last_peered":"2024-08-17T17:18:55.485166+0000","last_clean":"2024-08-17T17:18:55.485166+0000","last_became_active":"2024-08-17T17:05:34.072564+0000","last_became_peered":"2024-08-17T17:05:34.072564+0000","last_unstale":"2024-08-17T17:18:55.485166+0000","last_undegraded":"2024-08-17T17:18:55.485166+0000","last_fullsized":"2024-08-17T17:18:55.485166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:28.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.488468+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:18:55.488468+0000","last_peered":"2024-08-17T17:18:55.488468+0000","last_clean":"2024-08-17T17:18:55.488468+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:18:55.488468+0000","last_undegraded":"2024-08-17T17:18:55.488468+0000","last_fullsized":"2024-08-17T17:18:55.488468+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495009+0000","last_change":"2024-08-17T17:18:55.495009+0000","last_active":"2024-08-17T17:18:55.495009+0000","last_peered":"2024-08-17T17:18:55.495009+0000","last_clean":"2024-08-17T17:18:55.495009+0000","last_became_active":"2024-08-17T17:18:55.494532+0000","last_became_peered":"2024-08-17T17:18:55.494532+0000","last_unstale":"2024-08-17T17:18:55.495009+0000","last_undegraded":"2024-08-17T17:18:55.495009+0000","last_fullsized":"2024-08-17T17:18:55.495009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:54.695395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.489852+0000","last_change":"2024-08-17T17:17:50.474706+0000","last_active":"2024-08-17T17:18:54.489852+0000","last_peered":"2024-08-17T17:18:54.489852+0000","last_clean":"2024-08-17T17:18:54.489852+0000","last_became_active":"2024-08-17T17:17:50.474392+0000","last_became_peered":"2024-08-17T17:17:50.474392+0000","last_unstale":"2024-08-17T17:18:54.489852+0000","last_undegraded":"2024-08-17T17:18:54.489852+0000","last_fullsized":"2024-08-17T17:18:54.489852+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:38.744232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498410+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:18:55.498410+0000","last_peered":"2024-08-17T17:18:55.498410+0000","last_clean":"2024-08-17T17:18:55.498410+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:18:55.498410+0000","last_undegraded":"2024-08-17T17:18:55.498410+0000","last_fullsized":"2024-08-17T17:18:55.498410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494317+0000","last_change":"2024-08-17T17:18:55.493741+0000","last_active":"2024-08-17T17:18:55.494317+0000","last_peered":"2024-08-17T17:18:55.494317+0000","last_clean":"2024-08-17T17:18:55.494317+0000","last_became_active":"2024-08-17T17:18:55.493423+0000","last_became_peered":"2024-08-17T17:18:55.493423+0000","last_unstale":"2024-08-17T17:18:55.494317+0000","last_undegraded":"2024-08-17T17:18:55.494317+0000","last_fullsized":"2024-08-17T17:18:55.494317+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:15.411411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.497350+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:18:55.497350+0000","last_peered":"2024-08-17T17:18:55.497350+0000","last_clean":"2024-08-17T17:18:55.497350+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:18:55.497350+0000","last_undegraded":"2024-08-17T17:18:55.497350+0000","last_fullsized":"2024-08-17T17:18:55.497350+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.487419+0000","last_change":"2024-08-17T17:17:50.474242+0000","last_active":"2024-08-17T17:18:54.487419+0000","last_peered":"2024-08-17T17:18:54.487419+0000","last_clean":"2024-08-17T17:18:54.487419+0000","last_became_active":"2024-08-17T17:17:50.473923+0000","last_became_peered":"2024-08-17T17:17:50.473923+0000","last_unstale":"2024-08-17T17:18:54.487419+0000","last_undegraded":"2024-08-17T17:18:54.487419+0000","last_fullsized":"2024-08-17T17:18:54.487419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:13.899933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.493594+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:18:55.493594+0000","last_peered":"2024-08-17T17:18:55.493594+0000","last_clean":"2024-08-17T17:18:55.493594+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:18:55.493594+0000","last_undegraded":"2024-08-17T17:18:55.493594+0000","last_fullsized":"2024-08-17T17:18:55.493594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.499603+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:18:55.499603+0000","last_peered":"2024-08-17T17:18:55.499603+0000","last_clean":"2024-08-17T17:18:55.499603+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:18:55.499603+0000","last_undegraded":"2024-08-17T17:18:55.499603+0000","last_fullsized":"2024-08-17T17:18:55.499603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.495398+0000","last_change":"2024-08-17T17:18:55.495340+0000","last_active":"2024-08-17T17:18:55.495398+0000","last_peered":"2024-08-17T17:18:55.495398+0000","last_clean":"2024-08-17T17:18:55.495398+0000","last_became_active":"2024-08-17T17:18:55.495057+0000","last_became_peered":"2024-08-17T17:18:55.495057+0000","last_unstale":"2024-08-17T17:18:55.495398+0000","last_undegraded":"2024-08-17T17:18:55.495398+0000","last_fullsized":"2024-08-17T17:18:55.495398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:26.993423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498393+0000","last_change":"2024-08-17T17:18:55.498330+0000","last_active":"2024-08-17T17:18:55.498393+0000","last_peered":"2024-08-17T17:18:55.498393+0000","last_clean":"2024-08-17T17:18:55.498393+0000","last_became_active":"2024-08-17T17:18:55.496964+0000","last_became_peered":"2024-08-17T17:18:55.496964+0000","last_unstale":"2024-08-17T17:18:55.498393+0000","last_undegraded":"2024-08-17T17:18:55.498393+0000","last_fullsized":"2024-08-17T17:18:55.498393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:09:56.482828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482472+0000","last_change":"2024-08-17T17:18:22.904972+0000","last_active":"2024-08-17T17:18:54.482472+0000","last_peered":"2024-08-17T17:18:54.482472+0000","last_clean":"2024-08-17T17:18:54.482472+0000","last_became_active":"2024-08-17T17:18:22.904630+0000","last_became_peered":"2024-08-17T17:18:22.904630+0000","last_unstale":"2024-08-17T17:18:54.482472+0000","last_undegraded":"2024-08-17T17:18:54.482472+0000","last_fullsized":"2024-08-17T17:18:54.482472+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:01.926349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.494019+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:18:55.494019+0000","last_peered":"2024-08-17T17:18:55.494019+0000","last_clean":"2024-08-17T17:18:55.494019+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:18:55.494019+0000","last_undegraded":"2024-08-17T17:18:55.494019+0000","last_fullsized":"2024-08-17T17:18:55.494019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.484803+0000","last_change":"2024-08-17T17:17:19.406837+0000","last_active":"2024-08-17T17:18:55.484803+0000","last_peered":"2024-08-17T17:18:55.484803+0000","last_clean":"2024-08-17T17:18:55.484803+0000","last_became_active":"2024-08-17T17:17:19.406413+0000","last_became_peered":"2024-08-17T17:17:19.406413+0000","last_unstale":"2024-08-17T17:18:55.484803+0000","last_undegraded":"2024-08-17T17:18:55.484803+0000","last_fullsized":"2024-08-17T17:18:55.484803+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:22.497997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498573+0000","last_change":"2024-08-17T17:18:55.498573+0000","last_active":"2024-08-17T17:18:55.498573+0000","last_peered":"2024-08-17T17:18:55.498573+0000","last_clean":"2024-08-17T17:18:55.498573+0000","last_became_active":"2024-08-17T17:18:55.498230+0000","last_became_peered":"2024-08-17T17:18:55.498230+0000","last_unstale":"2024-08-17T17:18:55.498573+0000","last_undegraded":"2024-08-17T17:18:55.498573+0000","last_fullsized":"2024-08-17T17:18:55.498573+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.161705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.491060+0000","last_change":"2024-08-17T17:17:50.472168+0000","last_active":"2024-08-17T17:18:54.491060+0000","last_peered":"2024-08-17T17:18:54.491060+0000","last_clean":"2024-08-17T17:18:54.491060+0000","last_became_active":"2024-08-17T17:17:50.471864+0000","last_became_peered":"2024-08-17T17:17:50.471864+0000","last_unstale":"2024-08-17T17:18:54.491060+0000","last_undegraded":"2024-08-17T17:18:54.491060+0000","last_fullsized":"2024-08-17T17:18:54.491060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:32.866769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.492171+0000","last_change":"2024-08-17T17:18:55.492171+0000","last_active":"2024-08-17T17:18:55.492171+0000","last_peered":"2024-08-17T17:18:55.492171+0000","last_clean":"2024-08-17T17:18:55.492171+0000","last_became_active":"2024-08-17T17:18:55.491790+0000","last_became_peered":"2024-08-17T17:18:55.491790+0000","last_unstale":"2024-08-17T17:18:55.492171+0000","last_undegraded":"2024-08-17T17:18:55.492171+0000","last_fullsized":"2024-08-17T17:18:55.492171+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:44:21.883054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.490901+0000","last_change":"2024-08-17T17:05:34.066627+0000","last_active":"2024-08-17T17:18:54.490901+0000","last_peered":"2024-08-17T17:18:54.490901+0000","last_clean":"2024-08-17T17:18:54.490901+0000","last_became_active":"2024-08-17T17:05:34.066325+0000","last_became_peered":"2024-08-17T17:05:34.066325+0000","last_unstale":"2024-08-17T17:18:54.490901+0000","last_undegraded":"2024-08-17T17:18:54.490901+0000","last_fullsized":"2024-08-17T17:18:54.490901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:53.826538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.500485+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:18:55.500485+0000","last_peered":"2024-08-17T17:18:55.500485+0000","last_clean":"2024-08-17T17:18:55.500485+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:18:55.500485+0000","last_undegraded":"2024-08-17T17:18:55.500485+0000","last_fullsized":"2024-08-17T17:18:55.500485+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.498299+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:18:55.498299+0000","last_peered":"2024-08-17T17:18:55.498299+0000","last_clean":"2024-08-17T17:18:55.498299+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:18:55.498299+0000","last_undegraded":"2024-08-17T17:18:55.498299+0000","last_fullsized":"2024-08-17T17:18:55.498299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-17T17:18:54.482239+0000","last_change":"2024-08-17T17:18:22.903634+0000","last_active":"2024-08-17T17:18:54.482239+0000","last_peered":"2024-08-17T17:18:54.482239+0000","last_clean":"2024-08-17T17:18:54.482239+0000","last_became_active":"2024-08-17T17:18:22.903279+0000","last_became_peered":"2024-08-17T17:18:22.903279+0000","last_unstale":"2024-08-17T17:18:54.482239+0000","last_undegraded":"2024-08-17T17:18:54.482239+0000","last_fullsized":"2024-08-17T17:18:54.482239+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:08.309006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-17T17:18:55.483800+0000","last_change":"2024-08-17T17:18:22.902105+0000","last_active":"2024-08-17T17:18:55.483800+0000","last_peered":"2024-08-17T17:18:55.483800+0000","last_clean":"2024-08-17T17:18:55.483800+0000","last_became_active":"2024-08-17T17:18:22.901714+0000","last_became_peered":"2024-08-17T17:18:22.901714+0000","last_unstale":"2024-08-17T17:18:55.483800+0000","last_undegraded":"2024-08-17T17:18:55.483800+0000","last_fullsized":"2024-08-17T17:18:55.483800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:33.515070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201478,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36996,"kb_used_data":5880,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708156,"statfs":{"total":95995035648,"available":95957151744,"internally_reserved":0,"allocated":6021120,"data_stored":5413201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463116,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32896,"kb_used_data":1772,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712256,"statfs":{"total":95995035648,"available":95961350144,"internally_reserved":0,"allocated":1814528,"data_stored":1218505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724752,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32308,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712844,"statfs":{"total":95995035648,"available":95961952256,"internally_reserved":0,"allocated":1216512,"data_stored":624228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986388,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":5296,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":5423104,"data_stored":4822771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1180,"kb_used_omap":5,"kb_used_meta":27514,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1208320,"data_stored":623667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":28174409},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598806,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28720,"kb_used_data":1184,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716432,"statfs":{"total":95995035648,"available":95965626368,"internally_reserved":0,"allocated":1212416,"data_stored":624193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":28172461},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1176,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1204224,"data_stored":623323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":28172705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925859,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33404,"kb_used_data":5864,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711748,"statfs":{"total":95995035648,"available":95960829952,"internally_reserved":0,"allocated":6004736,"data_stored":5409126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":28172892},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:20:05.080 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:20:05.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump 2024-08-17T17:20:05.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:04 smithi175 ceph-mon[101434]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:05.299 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:05.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:05 smithi064 ceph-mon[109246]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:05.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:05 smithi064 ceph-mon[104516]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:05 smithi175 ceph-mon[101434]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:epoch 110 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:created 2024-08-17T16:58:44.445222+0000 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:modified 2024-08-17T17:18:55.480484+0000 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:crush_version 22 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:full_ratio 0.95 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:backfillfull_ratio 0.9 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:nearfull_ratio 0.85 2024-08-17T17:20:07.017 INFO:teuthology.orchestra.run.smithi064.stdout:require_min_compat_client luminous 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:min_compat_client luminous 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:require_osd_release reef 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:stretch_mode_enabled false 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:max_osd 8 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150] [v2:172.21.15.64:6804/2410633150,v1:172.21.15.64:6805/2410633150] exists,up 88904c1f-eac1-41db-b79a-1a4967d1dae9 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757] [v2:172.21.15.64:6812/3737445757,v1:172.21.15.64:6813/3737445757] exists,up 9d08169c-5914-4c23-acde-0607da1c3551 2024-08-17T17:20:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035] [v2:172.21.15.64:6820/3328029035,v1:172.21.15.64:6821/3328029035] exists,up 870e8cbe-170a-40f5-8482-de1f6ef4061e 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [26,105) [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413] [v2:172.21.15.64:6828/2515263413,v1:172.21.15.64:6829/2515263413] exists,up bad72588-3314-4260-ab5d-2b909812b54b 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 up in weight 1 up_from 31 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260] [v2:172.21.15.175:6802/3809583260,v1:172.21.15.175:6803/3809583260] exists,up d8a51704-ce10-47b2-806f-ee7a6c85aab0 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 up in weight 1 up_from 39 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320] [v2:172.21.15.175:6810/1423876320,v1:172.21.15.175:6811/1423876320] exists,up 1ebe3c8b-daa3-49aa-a503-4671f0f97c95 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 up in weight 1 up_from 47 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961] [v2:172.21.15.175:6818/916289961,v1:172.21.15.175:6819/916289961] exists,up 6bdf0ddc-f45b-4db6-9a7a-1769117b5b83 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 up in weight 1 up_from 55 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871] [v2:172.21.15.175:6826/3998219871,v1:172.21.15.175:6827/3998219871] exists,up 1d6aec1a-de13-4aa2-a0cf-5533286fcce6 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.4 [1,5] 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.f [1,2] 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.18 [1,2] 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/4035630069 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/57613042 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.019 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3398196717 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3794546409 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/267045601 expires 2024-08-18T16:59:13.244932+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3625104862 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3939830898 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/541572191 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2890874446 expires 2024-08-18T16:59:37.262677+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1135961297 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/1707444063 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:6834/2881274172 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1031286826 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/4139680086 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1209068324 expires 2024-08-18T16:59:13.244932+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3789040651 expires 2024-08-18T16:59:37.262677+0000 2024-08-17T17:20:07.020 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3939830898 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3929894796 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/1358035766 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3625104862 expires 2024-08-18T17:08:25.621451+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/327251329 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/2315698367 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2682323595 expires 2024-08-18T16:59:37.262677+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:6835/2881274172 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2830091420 expires 2024-08-18T16:59:13.244932+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2821917932 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3078753448 expires 2024-08-18T16:59:37.262677+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.175:0/2662913714 expires 2024-08-18T17:09:04.985494+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1140276429 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2705070003 expires 2024-08-18T17:11:24.558537+0000 2024-08-17T17:20:07.021 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2538331580 expires 2024-08-18T16:59:13.244932+0000 2024-08-17T17:20:07.022 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1209068324 expires 2024-08-18T16:59:13.244932+0000 2024-08-17T17:20:07.022 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1430444926 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.022 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1313240267 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.022 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3078753448 expires 2024-08-18T16:59:37.262677+0000 2024-08-17T17:20:07.022 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1140276429 expires 2024-08-18T17:04:10.165890+0000 2024-08-17T17:20:07.707 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.64:6802/2410633150,v1:172.21.15.64:6803/2410633150] [v2:172.21.15.64:6804/2410633150,v1:172.21.15.64:6805/2410633150] exists,up 88904c1f-eac1-41db-b79a-1a4967d1dae9', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.64:6810/3737445757,v1:172.21.15.64:6811/3737445757] [v2:172.21.15.64:6812/3737445757,v1:172.21.15.64:6813/3737445757] exists,up 9d08169c-5914-4c23-acde-0607da1c3551', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.64:6818/3328029035,v1:172.21.15.64:6819/3328029035] [v2:172.21.15.64:6820/3328029035,v1:172.21.15.64:6821/3328029035] exists,up 870e8cbe-170a-40f5-8482-de1f6ef4061e', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [26,105) [v2:172.21.15.64:6826/2515263413,v1:172.21.15.64:6827/2515263413] [v2:172.21.15.64:6828/2515263413,v1:172.21.15.64:6829/2515263413] exists,up bad72588-3314-4260-ab5d-2b909812b54b', 'osd.4 up in weight 1 up_from 31 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6800/3809583260,v1:172.21.15.175:6801/3809583260] [v2:172.21.15.175:6802/3809583260,v1:172.21.15.175:6803/3809583260] exists,up d8a51704-ce10-47b2-806f-ee7a6c85aab0', 'osd.5 up in weight 1 up_from 39 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6808/1423876320,v1:172.21.15.175:6809/1423876320] [v2:172.21.15.175:6810/1423876320,v1:172.21.15.175:6811/1423876320] exists,up 1ebe3c8b-daa3-49aa-a503-4671f0f97c95', 'osd.6 up in weight 1 up_from 47 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6816/916289961,v1:172.21.15.175:6817/916289961] [v2:172.21.15.175:6818/916289961,v1:172.21.15.175:6819/916289961] exists,up 6bdf0ddc-f45b-4db6-9a7a-1769117b5b83', 'osd.7 up in weight 1 up_from 55 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.175:6824/3998219871,v1:172.21.15.175:6825/3998219871] [v2:172.21.15.175:6826/3998219871,v1:172.21.15.175:6827/3998219871] exists,up 1d6aec1a-de13-4aa2-a0cf-5533286fcce6'] 2024-08-17T17:20:07.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon mon.a config show 2024-08-17T17:20:07.923 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:07.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:07 smithi064 ceph-mon[104516]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:07.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2408680491' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:20:07.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:07 smithi064 ceph-mon[109246]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:07.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2408680491' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:20:08.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:07 smithi175 ceph-mon[101434]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2408680491' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:20:09.473 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:20:09.473 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "mon.a", 2024-08-17T17:20:09.473 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster": "ceph", 2024-08-17T17:20:09.473 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket_mode": "", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_client_required": "cephx, none", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_cluster_required": "cephx", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_debug": "true", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_required": "cephx", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_ticket_ttl": "240.000000", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_supported": "", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio": "true", 2024-08-17T17:20:09.474 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_max_queue_depth": "1024", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_poll_ms": "250", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_reap_max": "16", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_async_discard_threads": "0", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_block_size": "4096", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio": "false", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_inflight_ios": "false", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_enable_discard": "false", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry": "3", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry_interval": "0.100000", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash": "0", 2024-08-17T17:20:09.475 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash_flush_delay": "2", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring": "false", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_hipri": "false", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_sqthread_poll": "false", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_buffer_alignment": "4096", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_type": "", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_alloc_size": "1048576", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_allocator": "hybrid", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_buffered_io": "true", 2024-08-17T17:20:09.476 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_for_zeros": "false", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_often": "false", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_compact_log_sync": "false", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_size": "16777216", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_replay_check_allocations": "true", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_log_runway": "4194304", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_prefetch": "1048576", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_flush_size": "524288", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_log_runway": "1048576", 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery": "false", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_shared_alloc_size": "65536", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_sync_write": "false", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocation_from_file": "true", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocator": "hybrid", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_async_db_compaction": "true", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bdev_label_multi": "true", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bdev_label_require_all": "true", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_blobid_prealloc": "10240", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_create": "true", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_create": "false", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_path": "", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_size": "0", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_path": "", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_preallocate_file": "false", 2024-08-17T17:20:09.479 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_size": "107374182400", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_create": "false", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_path": "", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_size": "100663296", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs": "true", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_env_mirror": "false", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-08-17T17:20:09.480 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune": "true", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size": "0", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-08-17T17:20:09.481 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_type": "2q", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_clone_cow": "true", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_algorithm": "snappy", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size": "0", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-08-17T17:20:09.482 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size": "0", 2024-08-17T17:20:09.485 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-08-17T17:20:09.485 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-08-17T17:20:09.485 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_mode": "none", 2024-08-17T17:20:09.485 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-08-17T17:20:09.485 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_csum_type": "crc32c", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_enforce_settings": "default", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_freelist": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_fsck_abort": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_read_err": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_legacy_omap": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-08-17T17:20:09.486 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_small_allocations": "0", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_read": "true", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_write": "false", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops": "0", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_elastic_shared_blobs": "true", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-08-17T17:20:09.487 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fail_eio": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs": "true", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-08-17T17:20:09.488 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_ignore_data_csum": "false", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-08-17T17:20:09.489 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kvbackend": "rocksdb", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_op_age": "5.000000", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_alloc_size": "0", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size": "0", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_defer_interval": "3.000000", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_deferred_txc": "32", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size": "0", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-08-17T17:20:09.490 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-08-17T17:20:09.491 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_nid_prealloc": "1024", 2024-08-17T17:20:09.491 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size": "0", 2024-08-17T17:20:09.491 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-08-17T17:20:09.491 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-08-17T17:20:09.491 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_qfsck_on_mount": "true", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_retry_disk_reads": "3", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cf": "true", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options_annex": "", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-08-17T17:20:09.492 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_coremask": "0x1", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_io_sleep": "5", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_max_io_completion": "0", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_mem": "512", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_sync_submit_transaction": "false", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_bytes": "67108864", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io": "0", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_tracing": "false", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-08-17T17:20:09.493 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved": "0", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_write_v2": "false", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_write_v2_random": "false", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_zero_block_detection": "false", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-08-17T17:20:09.494 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_action_update_interval": "2", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_mount_timeout": "10", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-08-17T17:20:09.495 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_signatures": "false", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_version": "2", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_signatures": "false", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_version": "2", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_signatures": "false", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_version": "2", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_sign_messages": "true", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "chdir": "", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_acl_type": "", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_asio_thread_count": "2", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_mid": "0.750000", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_size": "16384", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_caps_release_delay": "5", 2024-08-17T17:20:09.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_check_pool_perm": "true", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_collect_and_send_global_metrics": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_force_sync_read": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_getattr_caps": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_inject_features": "", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_inject_tick_delay": "0", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_remount": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_dirsize_rbytes": "true", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_force_lazyio": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_fs": "", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_fixed_oldest_tid": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_release_failure": "false", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_inline_size": "4096", 2024-08-17T17:20:09.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_retries_on_remount_failure": "5", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mds_namespace": "", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_metadata": "", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_gid": "-1", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_timeout": "300", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_uid": "-1", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mountpoint": "/", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_notify_timeout": "10", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc": "true", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty": "104857600", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty_age": "5.000000", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_objects": "1000", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_size": "209715200", 2024-08-17T17:20:09.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_target_dirty": "8388608", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_permissions": "true", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota": "true", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota_df": "true", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_bytes": "0", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_periods": "4", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_min": "131072", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_reconnect_stale": "false", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_shutdown_timeout": "30", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_snapdir": ".snap", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_tick_interval": "1", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_trace": "", 2024-08-17T17:20:09.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_try_dentry_invalidate": "false", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_faked_inos": "false", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_random_mds": "false", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog": "false", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_port": "12201", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_monitors": "default=true", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog": "false", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_level": "info", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_addr": "-", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network": "", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network_interface": "", 2024-08-17T17:20:09.500 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_isal": "false", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_level": "5", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_winsize": "-15", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zstd_level": "1", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_op_num_threads": "6", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_thread_cpu_cores": "", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_obc_lru_size": "512", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_stat_interval": "0", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_seastar_cpu_cores": "", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_seastar_num_threads": "0", 2024-08-17T17:20:09.501 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location": "", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook": "", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook_timeout": "10", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "daemonize": "false", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "daos_pool": "tank", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_name_prefix": "dbstore", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_alienstore": "0/5", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_allow_any_pool_priority": "false", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok": "1/5", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok_assert_abort": "false", 2024-08-17T17:20:09.502 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asserts_on_shutdown": "true", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_auth": "1/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bdev": "1/3", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluefs": "1/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluestore": "1/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_buffer": "0/1", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ceph_dedup": "0/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ceph_exporter": "1/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephfs_mirror": "0/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephsqlite": "0/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_client": "0/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_compressor": "1/5", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_context": "0/1", 2024-08-17T17:20:09.503 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crimson_interrupt": "0/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crush": "1/1", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crypto": "1/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cyanstore": "0/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_deliberately_leak_memory": "false", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_disable_randomized_ping": "false", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_dpdk": "1/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_eventtrace": "1/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filer": "0/1", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filestore": "1/3", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_finisher": "1/1", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_fuse": "1/5", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeat_testing_span": "0", 2024-08-17T17:20:09.504 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeatmap": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_immutable_obj_cache": "0/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_javaclient": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journal": "1/3", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journaler": "0/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_kstore": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_lockdep": "0/1", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mclock": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_balancer": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_locker": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log_expire": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_migrator": "1/5", 2024-08-17T17:20:09.505 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_quiesce": "3/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_memstore": "1/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgr": "2/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgrc": "1/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mon": "20/20", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_monc": "0/10", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ms": "1/1", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_none": "0/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objclass": "0/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objectcacher": "0/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objecter": "0/1", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_optracker": "0/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_osd": "1/5", 2024-08-17T17:20:09.506 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_paxos": "20/20", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_perfcounter": "1/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_prioritycache": "1/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rados": "0/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd": "0/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_mirror": "0/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_pwl": "0/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_replay": "0/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_refs": "0/0", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_reserver": "1/1", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw": "1/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_access": "1/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_datacache": "1/5", 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_dbstore": "1/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_flight": "1/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_lifecycle": "1/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_notification": "1/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_sync": "1/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rocksdb": "4/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_backref": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cache": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cleaner": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_device": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_epm": "0/5", 2024-08-17T17:20:09.508 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_journal": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_lba": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_odata": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_omap": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_onode": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_t": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_tm": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_striper": "0/1", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_test": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_throttle": "1/1", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_timer": "0/1", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_tp": "0/5", 2024-08-17T17:20:09.509 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_trace": "1/5", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "device_failure_prediction_mode": "none", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_graylog": "false", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_journald": "false", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_stderr": "true", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_syslog": "false", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event_tracing": "false", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_addr": "0.0.0.0", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_cert_file": "", 2024-08-17T17:20:09.510 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_http_port": "9926", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_key_file": "", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_prio_limit": "5", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sort_metrics": "true", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_stats_period": "5", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "fake_statfs_for_testing": "0", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "fatal_signal_handlers": "true", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_purge_ops": "10", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_truncate_ops": "128", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_apply_finisher_threads": "1", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_blackhole": "false", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_clone_range": "true", 2024-08-17T17:20:09.511 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_snap": "true", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_caller_concurrency": "10", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_collect_device_partition_information": "true", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_commit_timeout": "600.000000", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_inject_read_err": "false", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_omap_check": "false", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_verify_split": "false", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_dump_file": "", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fadvise": "true", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fail_eio": "true", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_shards": "16", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_size": "128", 2024-08-17T17:20:09.512 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap": "false", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap_threshold": "4096", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_index_retry_probability": "0.000000", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_inject_stall": "0", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_parallel": "false", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_trailing": "false", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_writeahead": "false", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_kill_at": "0", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size": "0", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-08-17T17:20:09.513 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs": "0", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_other": "2", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_sync_interval": "5.000000", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size": "0", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_merge_threshold": "-10", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_min_sync_interval": "0.010000", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_odsync_write": "false", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend": "rocksdb", 2024-08-17T17:20:09.514 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend_path": "", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_header_cache_size": "1024", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_ondisk_finisher_threads": "1", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_timeout": "60", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_threads": "2", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_punch_hole": "false", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_bytes": "104857600", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-08-17T17:20:09.515 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_ops": "50", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_seek_data_hole": "false", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc": "false", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_splice": "false", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_multiple": "2", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_rand_factor": "20", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_update_to": "1000", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-08-17T17:20:09.516 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_enable": "true", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_xfs_extsize": "false", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_zfs_snap": "false", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "fio_dir": "/tmp/fio", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "fsid": "d49b6e42-5cb9-11ef-bccf-c7b262605968", 2024-08-17T17:20:09.517 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_allow_other": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_atomic_o_trunc": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_big_writes": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_debug": "false", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_default_permissions": "false", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_disable_pagecache": "false", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_max_write": "0", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_multithreaded": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_require_active_mds": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_set_user_groups": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_move": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_read": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_write": "true", 2024-08-17T17:20:09.518 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_syncfs_on_mksnap": "true", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_use_invalidate_cb": "true", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_target_name": "ceph", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_file": "", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_inject_failure": "0", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_interval": "5", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "host": "smithi064", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_size": "1073741824", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-08-17T17:20:09.519 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_watermark": "0.900000", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "inject_early_sigterm": "false", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_agent_port": "6799", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_tracing_enable": "false", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_aio": "true", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_align_min_size": "65536", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_align": "true", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_size": "4096", 2024-08-17T17:20:09.520 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_dio": "true", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_discard": "false", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_force_aio": "false", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_ignore_corruption": "false", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_bytes": "10485760", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_entries": "100", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_replay_from": "0", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_multiple": "0.000000", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_max_multiple": "0.000000", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_write_header_frequency": "0", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_zero_on_create": "false", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prefetch_periods": "10", 2024-08-17T17:20:09.521 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prezero_periods": "5", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_write_head_interval": "15", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "keyfile": "", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_backend": "rocksdb", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_default_stripe_size": "65536", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount": "false", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount_deep": "true", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_bytes": "67108864", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_ops": "512", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_nid_prealloc": "1024", 2024-08-17T17:20:09.522 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_onode_map_size": "1024", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_submit_transaction": "false", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_transaction": "false", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "librados_thread_count": "2", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep": "false", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep_force_backtrace": "false", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_coarse_timestamps": "true", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_flush_on_exit": "false", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_host": "127.0.0.1", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_port": "12201", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_new": "1000", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_recent": "10000", 2024-08-17T17:20:09.523 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stderr_prefix": "", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stop_at_utilization": "0.970000", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_file": "true", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_graylog": "false", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_journald": "false", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_stderr": "false", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_syslog": "false", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "max_rotating_auth_attempts": "10", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_action_on_write_error": "1", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_alternate_name_max": "8192", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_asio_thread_count": "2", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_export_pin": "true", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_dirs": "true", 2024-08-17T17:20:09.524 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_interval": "5", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_size_max": "100000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_idle_threshold": "0.000000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_interval": "10", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max": "-1", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max_until": "-1", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_merge_size": "50", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_midchunk": "0.300000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_rebalance": "0.100000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_start": "0.200000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_minchunk": "0.001000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_mode": "0", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_max": "1.200000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_min": "0.800000", 2024-08-17T17:20:09.525 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_overload_epochs": "2", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_sample_interval": "3.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_bits": "3", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_rd": "25000.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_size": "10000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_wr": "10000.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_target_decay": "10.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_grace": "15.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_interval": "4.000000", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_mon_down_grace": "60", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_memory_limit": "4294967296", 2024-08-17T17:20:09.526 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_mid": "0.700000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_quiesce_delay": "0", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_quiesce_sleep": "200", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_quiesce_splitauth": "true", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_quiesce_threshold": "524288", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_release_free_interval": "10", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_reservation": "0.050000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_interval": "1", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_threshold": "262144", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-08-17T17:20:09.527 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_delegate_inos_pct": "50", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_prealloc_inos": "1000", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_connect_bootstrapping": "false", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_damage_table_max_entries": "10000", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_auth_pins": "false", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_frag": "false", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_scatterstat": "false", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_subtrees": "false", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_decay_halflife": "5.000000", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_default_dir_hash": "2", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_defer_session_stale": "true", 2024-08-17T17:20:09.528 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_deny_all_reconnect": "false", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_keys_per_op": "16384", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_commit_size": "10", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_entries": "0", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_prefetch": "true", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dirstat_min_interval": "1.000000", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_after_rejoin": "false", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_on_map": "false", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_file": "0", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_early_reply": "true", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enable_op_tracker": "true", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enforce_unique_name": "true", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed": "true", 2024-08-17T17:20:09.529 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random": "true", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_forward_all_requests_to_auth": "false", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_cache_threshold": "1.500000", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_summarize_threshold": "10", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_grace": "15.000000", 2024-08-17T17:20:09.530 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_reset_grace": "1000", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_health_dummy": "false", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_migrator_session_race": "false", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_join_fs": "", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_journal_format": "1", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_create_at": "0", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_dirfrag_at": "0", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_export_at": "0", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_import_at": "0", 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_expire_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_replay_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_link_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_mdstable_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_openc_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_rename_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_shutdown_at": "0", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_event_large_threshold": "524288", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_events_per_segment": "1024", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_major_segment_event_ratio": "12", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_events": "-1", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_segments": "128", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_pause": "false", 2024-08-17T17:20:09.532 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_segment_size": "0", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_skip_corrupt_events": "false", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_skip_unbounded_events": "false", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_trim_threshold": "128", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_warn_factor": "2.000000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_caps_per_client": "1048576", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_flushes": "100000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_requests": "100000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_export_size": "20971520", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_file_recover": "32", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_files": "64", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops": "8192", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-08-17T17:20:09.533 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_snaps_per_dir": "100", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_metrics_update_interval": "2", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_per_client": "100", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_working_set": "10000", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_numa_node": "-1", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_complaint_time": "30.000000", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_duration": "600", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_size": "20", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_slow_op_size": "20", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_slow_op_threshold": "10", 2024-08-17T17:20:09.534 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_log_threshold": "5", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_grace": "15", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_interval": "5", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_caps": "30000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_threshold": "131072", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_threshold": "262144", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_reconnect_timeout": "45.000000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_interval": "1.000000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-08-17T17:20:09.535 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_gid": "0", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_uid": "0", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_evict": "true", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_timeout": "true", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_metadata_threshold": "16777216", 2024-08-17T17:20:09.536 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_shutdown_check": "0", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_skip_ino": "0", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sleep_rank_change": "0.000000", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_max_uid": "4294967294", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_min_uid": "0", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_rstat": "false", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_standby_replay_damaged": "false", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_symlink_recovery": "true", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_task_status_update_interval": "2.000000", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_exports": "0", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_fragments": "0", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_tick_interval": "5.000000", 2024-08-17T17:20:09.537 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_valgrind_exit": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_backtrace": "1", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_scatter": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_ino_prealloc": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_sessions": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mempool_debug": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_debug_omit_block_device_write": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_device_bytes": "1073741824", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_set": "false", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_size": "65536", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_bytes": "134217728", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_messages": "512", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_connect_retry_interval": "1.000000", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_disabled_modules": "", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_enable_op_tracker": "true", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_max_pg_creating": "1024", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_max_pg_num_change": "128", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_bytes": "134217728", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_messages": "128", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_bytes": "134217728", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_messages": "128", 2024-08-17T17:20:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_num_op_tracker_shard": "32", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_complaint_time": "30.000000", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_history_duration": "600", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_history_size": "20", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_history_slow_op_size": "20", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_op_log_threshold": "5", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_bytes": "536870912", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_messages": "8192", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_pool": "true", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_service_beacon_grace": "60.000000", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_standby_modules": "true", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_period": "5", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_threshold": "5", 2024-08-17T17:20:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_tick_period": "2", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_accept_timeout_factor": "2.000000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_delete": "true", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_size_one": "false", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_auth_validate_all_caps": "true", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_bytes": "104857600", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_directed_command_retry": "2", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval": "3.000000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-08-17T17:20:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_parallel": "3", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_log_interval": "1.000000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_interval": "10.000000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_timeout": "30.000000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_target_rank": "-1", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_allowed": "1.000000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_level": "debug", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_file": "true", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog": "false", 2024-08-17T17:20:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_journald": "true", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_stderr": "false", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_bootstrap": "false", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_start": "false", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_trim": "true", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_persist_interval": "10", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_score_halflife": "43200", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_config_key_max_entry_size": "65536", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cpu_threads": "4", 2024-08-17T17:20:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_crush_min_required_version": "hammer", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_daemon_bytes": "419430400", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_crit": "5", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_warn": "5", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_size_warn": "16106127360", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_block_osdmap_trim": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_json": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_transactions": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_extra_checks": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-08-17T17:20:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_reef": "false", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_squid": "false", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_delta_reset_interval": "10.000000", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_dns_srv_name": "ceph-mon", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_down_mkfs_grace": "60", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_down_uptime_grace": "60", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_default_strategy": "3", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_timeout": "5.000000", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_divisor": "2", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_timeout": "2.000000", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_enable_op_tracker": "true", 2024-08-17T17:20:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_fake_pool_delete": "false", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_force_quorum_join": "false", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_fsmap_prune_threshold": "300", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_globalid_prealloc": "10000", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_detail_to_clog": "true", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_log_update_period": "5", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_max_detail": "50", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog": "true", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_interval": "600", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host": "[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host_override": "", 2024-08-17T17:20:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_initial_members": "", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_keyvaluedb": "rocksdb", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease": "5.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_full_interval": "50", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max": "10000", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max_summary": "50", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_entries_per_event": "4096", 2024-08-17T17:20:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_epochs": "500", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mdsmap_epochs": "500", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mgrmap_epochs": "500", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_nvmeof_epochs": "500", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_osd": "10000", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pg_per_osd": "10000", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pool_pg_num": "65536", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_force_trim_to": "0", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_skip_sanity": "true", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_autotune": "true", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_target": "2147483648", 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_beacon_grace": "30", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_digest_period": "5", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_inactive_grace": "60", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_mkfs_grace": "240", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_min_osdmap_epochs": "500", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_complaint_time": "30", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_duration": "600", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_size": "20", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_size": "20", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_threshold": "10", 2024-08-17T17:20:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_log_threshold": "5", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_in": "false", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_new_in": "true", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size": "500", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size_min": "134217728", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_crush_smoke_test": "true", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_destroyed_out_interval": "600", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_interval": "600", 2024-08-17T17:20:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_force_trim_to": "0", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_full_ratio": "0.950000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_halflife": "3600", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_max_interval": "300", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_weight": "0.300000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_max_initial_pgs": "1024", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_down_reporters": "2", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-08-17T17:20:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp": "true", 2024-08-17T17:20:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-08-17T17:20:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-08-17T17:20:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_report_timeout": "900", 2024-08-17T17:20:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_num_repaired": "10", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_op_age": "32.000000", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_interval": "10", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_min": "10000", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-08-17T17:20:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_stuck_threshold": "60", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_objects": "10000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_per_osd": "0", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_crit_threshold": "0", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_warn_threshold": "0", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_probe_timeout": "2.000000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_change": "0.050000", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_osds": "4", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-08-17T17:20:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_interval": "86400", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_max_keys": "100", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_timeout": "300", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_session_timeout": "300", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_smart_report_timeout": "5", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stat_smooth_intervals": "6", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_min_size": "2", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_size": "4", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-08-17T17:20:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_subscribe_interval": "86400.000000", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_debug": "false", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_keys": "2000", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_size": "1048576", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_provider_kill_at": "0", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_requester_kill_at": "0", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_timeout": "60.000000", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_target_pg_per_osd": "100", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_tick_interval": "5", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_interval": "300.000000", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_use_min_delay_socket": "false", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_older_version_delay": "604800", 2024-08-17T17:20:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_filestore_osds": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_misplaced": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_older_version": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_app": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-08-17T17:20:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_too_few_osds": "false", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "monmap": "", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-08-17T17:20:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_tracing_enabled": "false", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_op_threads": "3", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_buffer_size": "131072", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_cm": "false", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_device_name": "", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_dscp": "96", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_gid_idx": "0", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_local_gid": "", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_polling_us": "1000", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_port_num": "1", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_roce_ver": "1", 2024-08-17T17:20:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_send_buffers": "1024", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_sl": "3", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_support_srq": "true", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_type": "ib", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_reap_threshold": "5", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_before_connect": "false", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv4": "true", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv6": "false", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr1": "true", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr2": "true", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_max": "7568", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_min": "6800", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_prefer_ipv4": "false", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_count": "3", 2024-08-17T17:20:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_delay": "5", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_client": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mds": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mgr": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mon": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_osd": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_client_mode": "crc secure", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_mode": "crc secure", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_type": "", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_compress_secure": "false", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_idle_timeout": "900", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_ready_timeout": "10", 2024-08-17T17:20:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_data": "true", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_header": "true", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bad_msg": "false", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bug": "true", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_old_message": "true", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_skipped_message": "false", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_unhandled_msg": "false", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_coremask": "0xF", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_devs_allowlist": "", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_enable_tso": "true", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-08-17T17:20:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hugepages": "", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_flow_control": "true", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_lro": "true", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_memory_channel": "4", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_pmd": "", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_port_id": "0", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_corrupt_message_level": "1", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_on_send": "false", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_initial_backoff": "0.200000", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_max": "1.000000", 2024-08-17T17:20:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_probability": "0.000000", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_type": "", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_internal_delays": "0.000000", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_network_congestion": "0", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_socket_failures": "0", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_learn_addr_from_peer": "true", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_accept_failures": "4", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_backoff": "15.000000", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_client_mode": "secure crc", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_cluster_mode": "secure crc", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_service_mode": "secure crc", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_min_size": "1024", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_mode": "none", 2024-08-17T17:20:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compression_algorithm": "snappy", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_min_cost": "65536", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_public_type": "", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_service_mode": "crc secure", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_listen_backlog": "512", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_nodelay": "true", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_rcvbuf": "0", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_time_events_min_wait_interval": "1000", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_type": "async+posix", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "no_config_file": "false", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-08-17T17:20:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "nvmeof_mon_client_tick_period": "2", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_completion_locks_per_session": "32", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_debug_inject_relock_delay": "false", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_op_bytes": "104857600", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_ops": "1024", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inject_no_watch_ping": "false", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_tick_interval": "5.000000", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_timeout": "10.000000", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objectstore_blackhole": "false", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "openssl_engine_opts": "", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_delay_time": "5.000000", 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_hist_halflife": "1000", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_low_ops": "2", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_ops": "4", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_quantize_effort": "0.100000", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_slop": "0.020000", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_allow_recovery_below_min_size": "true", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_asio_thread_count": "2", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_async_recovery_min_cost": "100", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_auto_mark_unfound_lost": "false", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_retry_interval": "30.000000", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_max": "512", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_min": "64", 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_degraded": "false", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_peering": "false", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_unfound": "true", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_beacon_report_interval": "300", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_duration": "30", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_max_block_size": "67108864", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_small_size_max_iops": "100", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blkin_trace_all": "false", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blocked_scrub_grace_period": "120", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-08-17T17:20:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_for_log_corruption": "false", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_update_on_start": "true", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_cap": "256", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_size_cap": "524288000", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_op_priority": "63", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_watch_timeout": "30", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_command_max_records": "256", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_compact_on_start": "false", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-08-17T17:20:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_chooseleaf_type": "1", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_initial_weight": "-1.000000", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_on_start": "true", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_weight_set": "true", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_duration": "0", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_feed_pullee": "-1", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_misdirected_ops": "false", 2024-08-17T17:20:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_acting_change": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_purge_strays": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_op_order": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pg_log_writeout": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pretend_recovery_active": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_shutdown": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_trim_objects": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_cached_snaps": "false", 2024-08-17T17:20:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_missing_on_start": "false", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_snaps": "false", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_keys": "1024", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_stride": "524288", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_data_pool_replay_window": "45", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_notify_timeout": "30", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep": "0.000000", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-08-17T17:20:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_discard_disconnected_ops": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_ec_partial_reads": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_enable_op_tracker": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_extblkdev_plugins": "vdo", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_info": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_timeout": "15", 2024-08-17T17:20:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_function_tracing": "false", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_grace": "20", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_interval": "6", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_peers": "10", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_size": "2000", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_stale": "600", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_max_size": "100000", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_min_size": "1000", 2024-08-17T17:20:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_flush_on_shutdown": "true", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_size": "5120", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_kill_backfill_at": "0", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_loop_before_reset_tphandle": "64", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_cache_size": "50", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_dedup": "true", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max": "40", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max_bytes": "10485760", 2024-08-17T17:20:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_share_max_epochs": "40", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_name_len": "100", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_size": "0", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_backfills": "1", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_count": "5", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_period": "600", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_name_len": "2048", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_namespace_len": "256", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_size": "134217728", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_entries_per_request": "1024", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_blocked_by": "16", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_log_entries": "10000", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-08-17T17:20:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pgls": "1024", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_cost": "8388608", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_objects": "10", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_scrubs": "3", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_trimming_pgs": "2", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_op_reply_len": "64", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_size": "90", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-08-17T17:20:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_override_recovery_settings": "false", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_profile": "balanced", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-08-17T17:20:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_skip_benchmark": "false", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_base": "805306368", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_min": "134217728", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target": "4294967296", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_autotune": "false", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_pg_log_entries": "250", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_recovery_priority": "0", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_interval": "30", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-08-17T17:20:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_interval": "5", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_max_in_flight": "2", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_cache_shards": "32", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_op_tracker_shard": "32", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_auto_affinity": "true", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_node": "-1", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_prefer_iface": "true", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objecter_finishers": "1", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore": "bluestore", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_fuse": "false", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_ideal_list_max": "64", 2024-08-17T17:20:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_tracing": "false", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_complaint_time": "30.000000", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_duration": "600", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_size": "20", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_size": "20", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_log_threshold": "5", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards": "0", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_hdd": "5", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_ssd": "8", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard": "0", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-08-17T17:20:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_min_cost": "65536", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue": "mclock_scheduler", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue_cut_off": "high", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_suicide_timeout": "150", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_timeout": "15", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_open_classes_on_start": "true", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_os_flags": "0", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_peering_op_priority": "255", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_cost": "1048576", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_priority": "5", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-08-17T17:20:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_dups_tracked": "3000", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_max": "10000", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_min": "100", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_object_context_cache_count": "64", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-08-17T17:20:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crimson": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crush_rule": "-1", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_ec_fast_read": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_bulk": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nodelete": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flags": "0", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_min_size": "0", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-08-17T17:20:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_num": "32", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pgp_num": "0", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_read_ratio": "70", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_size": "3", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_type": "replicated", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_use_gmt_hitset": "true", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_push_per_object_cost": "1000", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_read_ec_check_for_errors": "false", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap": "true", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap_limit": "10", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_cost": "20971520", 2024-08-17T17:20:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_delay_start": "0.000000", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active": "0", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_hdd": "3", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_ssd": "10", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_chunk": "8388608", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_single_start": "1", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_priority": "3", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_warn_multiple": "16", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_priority": "5", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_retry_interval": "30.000000", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep": "0.000000", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-08-17T17:20:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_repair_during_recovery": "false", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_requested_scrub_priority": "120", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rollback_to_cluster_snap": "", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair": "false", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_hour": "0", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_week_day": "0", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_max": "25", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_min": "5", 2024-08-17T17:20:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_cost": "52428800", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_during_recovery": "false", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_hour": "0", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_week_day": "0", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_event_cost": "4096", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_invalid_stats": "true", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_load_threshold": "0.500000", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_interval": "604800.000000", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_preemptions": "5", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_min_interval": "86400.000000", 2024-08-17T17:20:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_priority": "5", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_sleep": "0.000000", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shutdown_pgref_assert": "false", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_skip_data_digest": "false", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_smart_report_timeout": "5", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_cost": "1048576", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_priority": "5", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep": "0.000000", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-08-17T17:20:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_scrubbing": "15", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_transaction_size": "30", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_mode": "writeback", 2024-08-17T17:20:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tracing": "false", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_use_stale_snap": "false", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "osdc_blkin_trace_all": "false", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_kill_at": "0", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_max_join_drift": "10", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min": "500", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min_wait": "0.050000", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_propose_interval": "1.000000", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max": "500", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max_multiplier": "20", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_min": "250", 2024-08-17T17:20:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_stash_full_interval": "25", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_max": "500", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_min": "250", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "perf": "true", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "pid_file": "", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addr": "-", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addrv": "", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "public_bind_addr": "-", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network": "172.21.0.0/20", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network_interface": "", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_busy_polling": "false", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_enabled": "false", 2024-08-17T17:20:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_session_max_number": "256", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_mon_op_timeout": "0", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_osd_op_timeout": "0", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_tracing": "false", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_atime_update_interval": "60", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_parent_reads": "false", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_snap_reads": "false", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blkin_trace_all": "false", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_expire_seconds": "0", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_on_break_lock": "true", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache": "true", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_block_writes_upfront": "false", 2024-08-17T17:20:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty": "25165824", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_object": "0", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_policy": "writearound", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_size": "33554432", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_target_dirty": "16777216", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_clone_copy_on_read": "false", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_compression_hint": "none", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_concurrent_management_ops": "10", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_clone_format": "auto", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_data_pool": "", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_features": "61", 2024-08-17T17:20:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_format": "2", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_map_options": "", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_order": "22", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_pool": "rbd", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_count": "0", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_unit": "0", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_disable_zero_copy_writes": "true", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_granularity_bytes": "65536", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_enable_alloc_hint": "true", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler": "simple", 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_commit_age": "5.000000", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_interval": "0", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_order": "24", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_pool": "", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_splay_width": "4", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_parent_reads": "false", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_snap_reads": "false", 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_type": "simple", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-08-17T17:20:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_autotune": "true", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_base": "805306368", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_target": "4294967296", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_delete_delay": "0", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_replay_delay": "0", 2024-08-17T17:20:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove": "false", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mtime_update_interval": "60", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_non_blocking_aio": "true", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_thread_timeout": "60", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_threads": "1", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_parent_cache_enabled": "false", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_mode": "disabled", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_path": "/tmp", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_size": "1073741824", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_plugins": "", 2024-08-17T17:20:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_limit": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_exclude_ops": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_limit": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_limit": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_limit": "0", 2024-08-17T17:20:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_schedule_tick_min": "50", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst": "0", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_limit": "0", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst": "0", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_limit": "0", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_quiesce_notification_attempts": "10", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_read_from_replica_policy": "default", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_max_bytes": "524288", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_trigger_requests": "10", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_request_timed_out_seconds": "30", 2024-08-17T17:20:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_skip_partial_discard": "true", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_tracing": "false", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_names": "true", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_pool": "true", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_base_url": "", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_log_level": "", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_account_default_quota_max_size": "-1", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_acl_grants_max_num": "100", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_admin_entry": "admin", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_asio_assert_yielding": "false", 2024-08-17T17:20:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_backend_store": "rados", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_barbican_url": "", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_beast_enable_async": "true", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_counters_cache": "false", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_max_aio": "128", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_ttl": "600", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_enabled": "true", 2024-08-17T17:20:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_expiry_interval": "900", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_lru_size": "25000", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_config_store": "rados", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_content_length_compat": "false", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress": "true", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cors_rules_max_num": "100", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cross_domain_policy": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_default_encryption_key": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_addr": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_ca_path": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_cert": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_key": "", 2024-08-17T17:20:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_password": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_username": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_require_ssl": "true", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-08-17T17:20:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_suppress_logs": "true", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_addr": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_auth": "token", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_namespace": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_prefix": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-08-17T17:20:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_token_file": "", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_buffersize": "524288", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_limit": "1024", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_time": "300", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_tcp_keepalive": "0", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_fadvise": "4", 2024-08-17T17:20:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_l1_fadvise": "4", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_changes_size": "1000", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_num_shards": "128", 2024-08-17T17:20:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_window": "30", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_notify_interval_msec": "0", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_poll_interval": "20", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_spawn_window": "20", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_data_log_backing": "fifo", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_region_info_oid": "default.region", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-08-17T17:20:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_defer_to_bucket_acls": "", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_disable_s3select": "false", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_res": "500.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-08-17T17:20:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_name": "", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_s3website_name": "", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dynamic_resharding": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_gc_threads": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_lc_threads": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_mdsearch": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_ops_log": "false", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_quota_threads": "true", 2024-08-17T17:20:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_static_website": "false", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_usage_log": "false", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enforce_swift_acls": "true", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_exit_timeout_secs": "120", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_expose_bucket": "false", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_extended_http_attrs": "", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_filter": "none", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontends": "beast port=7480", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_concurrent_io": "10", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred": "50", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_objs": "32", 2024-08-17T17:20:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_queue_size": "134213632", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_trim_chunk": "16", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_obj_min_wait": "7200", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_max_time": "3600", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_period": "3600", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_window_size": "16777216", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_healthcheck_disabling_path": "", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ignore_get_invalid_range": "false", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_init_timeout": "300", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_kafka_connection_idle": "300", 2024-08-17T17:20:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_kafka_message_timeout": "5000", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_kafka_sleep_timeout": "10", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_domain": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password_path": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_project": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_tenant": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token_path": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_user": "", 2024-08-17T17:20:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_api_version": "2", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_domain": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_password": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_project": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_tenant": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_user": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_implicit_tenants": "false", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_enabled": "false", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_token_cache_size": "10000", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_url": "", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_verify_ssl": "true", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_debug_interval": "-1", 2024-08-17T17:20:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_lock_max_time": "90", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_objs": "32", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_rules": "1000", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_worker": "3", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_wp_worker": "3", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_thread_delay": "0", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_dnattr": "uid", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchfilter": "", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_uri": "", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lfuda_sync_frequency": "60", 2024-08-17T17:20:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_http_headers": "", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_nonexistent_bucket": "false", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name_utc": "false", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_name_len": "0", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_size": "0", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attrs_num_in_req": "0", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_chunk_size": "4194304", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_concurrent_requests": "1024", 2024-08-17T17:20:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_dynamic_shards": "1999", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_listing_results": "1000", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_notify_retries": "10", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_objs_per_shard": "100000", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_param_size": "1048576", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_size": "5368709120", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_slo_entries": "1000", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_log_max_shards": "64", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_notify_interval_msec": "200", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_poll_interval": "20", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_spawn_window": "20", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-08-17T17:20:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mp_lock_max_time": "600", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_min_part_size": "5242880", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_size": "2017", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lanes": "5", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_max_gc": "300", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_gc_threads": "false", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_lc_threads": "false", 2024-08-17T17:20:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_quota_threads": "false", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_sync_thread": "false", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_async_rados_threads": "32", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_control_oids": "8", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_numa_node": "-1", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_stripe_size": "4194304", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_chunk_size": "100", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_gc_interval": "600", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_hints_num_shards": "127", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-08-17T17:20:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_timeout": "600", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_tracing": "false", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_token": "", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_url": "", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_verify_ssl": "true", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_rados": "false", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_socket_path": "", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-08-17T17:20:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval": "2.000000", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval_max": "30.000000", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_root_pool": ".rgw.root", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_cache_lanes": "3", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_cache_max_buckets": "100", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_cache_partitions": "3", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_continue": "true", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_prohibited_content_length": "false", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_tracing": "false", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm": "", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_id": "", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region": "", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region_root_pool": ".rgw.root", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_region_enforcement": "false", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_request_uri": "", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_batch_size": "64", 2024-08-17T17:20:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_debug_interval": "-1", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_max_aio": "128", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_num_logs": "16", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_thread_interval": "600", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_resolve_cname": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rest_getusage_op_compat": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_run_sync_thread": "true", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_keystone": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_ldap": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_rados": "true", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_sts": "false", 2024-08-17T17:20:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_success_create_obj_status": "0", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_scheduler_type": "throttler", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_script_uri": "", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_service_provider_name": "", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_id": "", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_secret": "", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_entry": "sts", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_key": "sts", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_max_session_duration": "43200", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_min_session_duration": "900", 2024-08-17T17:20:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_token_introspection_url": "", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_account_in_url": "false", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_entry": "auth", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_url": "", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_custom_header": "", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_enforce_content_length": "false", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_need_stats": "true", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_tenant_name": "", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_token_expiration": "86400", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url": "", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url_prefix": "swift", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_versioning_enabled": "false", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-08-17T17:20:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_lease_period": "120", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_interval": "1200", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_obj_etag_verify": "false", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_history_size": "4096", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_thread_pool_size": "512", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_topic_persistency_max_retries": "0", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-08-17T17:20:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_topic_require_publish_policy": "false", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_comment": "", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_createby": "", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_encoding": "", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_flag": "false", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_max_size": "5368709120", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_origin": "", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_sha_unit": "524288", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_tracker": "", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_trust_forwarded_https": "false", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_tick_interval": "30", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_shards": "32", 2024-08-17T17:20:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_user_shards": "1", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_use_opa_authz": "false", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_counters_cache": "false", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_counters_cache_size": "10000", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_size": "-1", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_max_buckets": "1000", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_policies_max_num": "100", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_interval": "86400", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_unique_email": "true", 2024-08-17T17:20:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_verify_ssl": "true", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_website_routing_rules_max_num": "50", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone": "", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_id": "", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup": "", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_id": "", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_auto_session_clear": "true", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_skip_bucket_step": "false", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_block_size": "4096", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-08-17T17:20:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_shard_bits": "4", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_size": "536870912", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_type": "binned_lru", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_compaction_stats": "false", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_extended_stats": "false", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_memory_stats": "false", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_index_type": "binary_search", 2024-08-17T17:20:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_log_to_ceph_log": "true", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_metadata_block_size": "4096", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_partition_filters": "false", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_perf": "false", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_renewal_timeout": "10", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "run_dir": "/var/run/ceph", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_block_create": "true", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cache_lru_size": "67108864", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cbjournal_size": "5368709120", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_data_delta_based_overwrite": "0", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_max_object_size": "16777216", 2024-08-17T17:20:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_device_size": "53687091200", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_full_integrity_check": "false", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_capacity": "16", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_iodepth_limit": "5", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_main_device_type": "SSD", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_max_concurrent_transactions": "8", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_max_data_allocation_size": "32768", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-08-17T17:20:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_segment_size": "67108864", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "set_keepcaps": "false", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "setgroup": "ceph", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser": "ceph", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser_match_path": "", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "target_max_misplaced_ratio": "0.050000", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "thp": "false", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_default_timeout": "60", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_empty_queue_max_wait": "2", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "throttler_perf_counter": "true", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "tmp_dir": "/tmp", 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX" 2024-08-17T17:20:09.634 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:20:10.059 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:09 smithi064 ceph-mon[104516]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:10.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:09 smithi064 ceph-mon[109246]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:09 smithi175 ceph-mon[101434]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:10.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-08-17T17:20:10.738 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:11.095 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:11.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:12.073 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:11 smithi064 ceph-mon[104516]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:12.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:11 smithi064 ceph-mon[109246]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:12.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:11 smithi175 ceph-mon[101434]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:12.395 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2024-08-17T17:20:12.395 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:20:12.405 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2024-08-17T17:20:12.405 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:20:12.414 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2024-08-17T17:20:12.414 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:20:13.012 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-08-17T17:20:13.012 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T17:20:13.013 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:20:13.015 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:20:13.015 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-08-17T17:20:13.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-08-17T17:20:13.016 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-08-17T17:20:13.016 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:13.016 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:13.016 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-17T17:20:13.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-17T17:20:13.017 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-08-17T17:20:13.017 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:13.017 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-08-17T17:20:13.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:13.018 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-08-17T17:20:13.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:20:13.018 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-08-17T17:20:13.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:20:13.046 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:13.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4043108096' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:20:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:13.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4043108096' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:20:13.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:13.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4043108096' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:20:13.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:13.276 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:13.281 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:13.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:13.287 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:13.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:13.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:13.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:13.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:13.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:13.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:13.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:13.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:13.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:13.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:13.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:13 smithi064 ceph-mon[104516]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:13.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:13 smithi064 ceph-mon[109246]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:13.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:14.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:14.070 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:14.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:13 smithi175 ceph-mon[101434]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:14.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:14.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:14.230 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:13] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-08-17T17:20:14.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:14.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:14.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:14.449 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:14.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:14.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:14.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:14.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:14.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:14.698 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:14.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:14.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:14.826 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:14.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:14.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.055 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:15.080 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:15.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:15.306 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:15.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:15.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:15.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:15.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:15.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:15.745 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:15.745 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:15.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:15.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:15.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:15.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:15 smithi064 ceph-mon[104516]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:15.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:15 smithi064 ceph-mon[109246]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:15.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:16.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:16.086 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:16.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:15 smithi175 ceph-mon[101434]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:16.188 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:16.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:16.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:16.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:16.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:16.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:16.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:16.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:16.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:16.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:16.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:16.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:16.835 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:16.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:16.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:17.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:17.089 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:17.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:17.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:17.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:17.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:17.360 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:20:17.361 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:20:17.361 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2024-08-17T17:20:17.361 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:20:17.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:17.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:17.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:17.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:17.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:17.713 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:17.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:17.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:17.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:17.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:18.064 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:18.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:17 smithi064 ceph-mon[104516]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:18.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:17 smithi064 ceph-mon[109246]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:18.089 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:18.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:18.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:17 smithi175 ceph-mon[101434]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:18.191 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:18.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:18.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:18.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:18.464 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:18.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:18.589 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:18.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:18.713 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:18.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:18.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:18.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:18.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.064 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:19.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:19.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:19.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:19.330 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:20:19.331 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:19.342 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:20:19.342 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:19.343 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:19.358 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:20:19.359 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:19.374 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:20:19.375 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:19.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:20:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:19.408 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:20:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:19.425 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:20:19.425 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:20:19.443 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:19.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:19.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:19.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:19.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:19 smithi064 ceph-mon[104516]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2062106081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:19.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:19 smithi064 ceph-mon[109246]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:19.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2062106081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:19.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:19.966 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:20.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:20.094 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:20.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:20.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:19 smithi175 ceph-mon[101434]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2062106081' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:20.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:20.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:20.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:20.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:20.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.474 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:20.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:20.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:20.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:20.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:20.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:20.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:21.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:21.104 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:21.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:21.229 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:21.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:21.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:21.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:21.480 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:21.582 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:21.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:20:21.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:20:21.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:21.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:21.786 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' created 2024-08-17T17:20:21.790 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:20:21.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:21.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:21.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:21.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[109246]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:21.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:20:21.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2548447735' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:21.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[104516]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:20:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2548447735' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:21.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:21.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:21.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:22.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:22.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:21 smithi175 ceph-mon[101434]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:20:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2548447735' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:22.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:22.230 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:22.332 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:22.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:22.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:22.481 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:22.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:22.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:22.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:20:22.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:22.735 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:22.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:22.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:22.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:22.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:20:22.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:22.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[104516]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:20:22.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:22.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:20:22.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:22.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:22 smithi064 ceph-mon[109246]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:20:22.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:22.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-08-17T17:20:22.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:22.997 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:23.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:23.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:23.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:22 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:22 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3142539519' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:20:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:22 smithi175 ceph-mon[101434]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:20:23.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:23.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:23.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:23.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:23.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:23.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:23.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:23.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:23.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:23.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:23.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:23.876 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:23.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[104516]: pgmap v293: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:23.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:23.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[104516]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:20:23.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[109246]: pgmap v293: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:23.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:23.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-mon[109246]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:20:23.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:23.997 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:23.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-17T17:20:24.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:24.090 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:24.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:24.117 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:24.124 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:24.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:24.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:23 smithi175 ceph-mon[101434]: pgmap v293: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:24.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:23 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:23 smithi175 ceph-mon[101434]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:20:24.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:23] "GET /metrics HTTP/1.1" 200 34149 "" "Prometheus/2.51.0" 2024-08-17T17:20:24.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:24.359 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:24.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:24.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:24.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:24.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:24.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:24.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:24.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:24.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:25.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:25.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:24 smithi064 ceph-mon[104516]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:20:25.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:24 smithi064 ceph-mon[109246]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:20:25.105 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:24 smithi175 ceph-mon[101434]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:20:25.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:25.258 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:25.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:25.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:20:25.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:25.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:25.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:25.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:25.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:25.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:25.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:25.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:25.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:25.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:26.017 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:26.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:26.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[109246]: pgmap v296: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[104516]: pgmap v296: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:26.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:26.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:26.328 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:26 smithi175 ceph-mon[101434]: pgmap v296: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:26.328 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:26.328 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:26.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:26.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:26.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:26.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:26.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:26.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:26.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:26.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:26.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:26.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:26.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:26.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:26.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:26.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:27.012 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:27.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:27.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:27.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:27.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:27.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:27.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:27.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:27.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:27.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:27.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:27.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:27.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:27.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:20:27.679 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:27.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:27.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:20:27.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:27.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:20:27.723 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:27.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:27.740 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:20:27.740 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:27.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:27.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:20:27.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:27.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:20:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:27.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:20:27.790 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:27.806 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:20:27.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:27.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:27.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:27.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:28.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:28.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:28.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:28.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:28.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:28.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:28.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:28.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:28 smithi175 ceph-mon[101434]: pgmap v297: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:28.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1963515214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:28 smithi064 ceph-mon[104516]: pgmap v297: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1963515214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:28 smithi064 ceph-mon[109246]: pgmap v297: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:28.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1963515214' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:28.499 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:28.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:28.622 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:28.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:28.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:28.874 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:28.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:28.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:28.998 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:29.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:29.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:29.151 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:20:29.151 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:20:29.151 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:20:29.151 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:20:29.151 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:20:29.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:29.249 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:29.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:29.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:29.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:29.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:29.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:29.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:29.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:29.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:29.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:29.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:29.877 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:29.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:29.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:30.004 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:30.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:30 smithi064 ceph-mon[104516]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:30.125 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:30 smithi064 ceph-mon[109246]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:30.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:30.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:30.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:30.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:30 smithi175 ceph-mon[101434]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:30.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:30.504 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:30.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:30.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:30.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:30.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:30.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:31.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:31.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:31.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:31.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:31.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:31.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:31.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:31.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:31.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:31.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:31.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:31.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:31.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:31.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:31.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:31.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:31.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:31.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:31.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:32.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:32.047 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:20:32.052 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-08-17T17:20:32.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:32.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:32.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:32.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:32.359 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:32.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[104516]: pgmap v299: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4129186211' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[109246]: pgmap v299: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4129186211' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:32.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:32 smithi175 ceph-mon[101434]: pgmap v299: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4129186211' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:20:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:32.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:32.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:32.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:32.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:32.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:32.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:32.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:32.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:32.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:33.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:33.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:33.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:33.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:20:33.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[109246]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:20:33.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:33.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:20:33.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-mon[104516]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:20:33.234 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:33.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:33.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:33.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:20:33.362 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:33.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:33 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:20:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3292766259' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:33 smithi175 ceph-mon[101434]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:20:33.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:33.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:33.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:33.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:33.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:33.823 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-08-17T17:20:33.825 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:20:33.825 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:20:33.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.878 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:20:33.878 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:20:33.878 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_147470 2024-08-17T17:20:33.878 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:20:33.879 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:20:33.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:33.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:34.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:34.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:34.138 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:34.171 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:34 smithi064 ceph-mon[104516]: pgmap v301: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:34 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:33] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-08-17T17:20:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:34 smithi064 ceph-mon[109246]: pgmap v301: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:34 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:34.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:34.311 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:34 smithi175 ceph-mon[101434]: pgmap v301: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:34 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:34.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:34.441 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:34.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:34.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:34.559 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:20:34.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:20:34.568 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:34.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:34.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:34.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:34.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:34.879 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3641 3625 226.549 226.562 0.00442102 0.00440157 2024-08-17T17:20:34.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:34.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:35.017 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:35.118 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:35.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:35.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:35.291 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:35.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:35.435 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:35.436 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:35.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-17T17:20:35.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:35.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:35.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:35.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:35.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:35.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:35.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:35.878 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:35.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 7422 7406 231.236 236.312 0.00790357 0.00431165 2024-08-17T17:20:35.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:35.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:20:35.985 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:35.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:36.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:36.133 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:36.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:36.274 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:36.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:36 smithi175 ceph-mon[101434]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:36.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:36 smithi064 ceph-mon[104516]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:36.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:36 smithi064 ceph-mon[109246]: pgmap v302: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:20:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:36.483 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:36.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:36.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:36.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:36.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:36.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:36.839 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:36.880 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 10797 10781 224.467 210.938 0.00480923 0.00444848 2024-08-17T17:20:36.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:36.984 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:37.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:37.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:37.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:37.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:37.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:37.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:37.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:37.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:37.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:37.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:37.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:37.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:37.880 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 14208 14192 221.643 213.188 0.0040971 0.00450505 2024-08-17T17:20:37.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:37.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:38.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:38.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:38.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:38.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:38.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:38.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:38 smithi175 ceph-mon[101434]: pgmap v303: 89 pgs: 89 active+clean; 331 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2024-08-17T17:20:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:38 smithi064 ceph-mon[104516]: pgmap v303: 89 pgs: 89 active+clean; 331 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2024-08-17T17:20:38.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:38 smithi064 ceph-mon[109246]: pgmap v303: 89 pgs: 89 active+clean; 331 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s 2024-08-17T17:20:38.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:38.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:38.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:38.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:38.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 17016 17000 212.412 175.5 0.00580526 0.00470132 2024-08-17T17:20:38.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:38.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:38.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:39.036 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:39.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:39.176 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:39.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:39.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:39.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:39.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:39.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:39.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:39.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:39.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:39.828 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:39.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:39.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 19906 19890 207.096 180.625 0.00386129 0.00482147 2024-08-17T17:20:39.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.063 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:20:40.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:40.102 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:20:40.103 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:40.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:40.141 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:20:40.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:40.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:40.176 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:20:40.177 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:40.217 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:20:40.217 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:40.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:20:40.243 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:40.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.260 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:20:40.260 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:40.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.282 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:20:40.282 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:40.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[104516]: pgmap v304: 89 pgs: 89 active+clean; 551 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-08-17T17:20:40.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3389382634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:40.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:40.283 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[109246]: pgmap v304: 89 pgs: 89 active+clean; 551 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-08-17T17:20:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3389382634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:40.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.405 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:40.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:40 smithi175 ceph-mon[101434]: pgmap v304: 89 pgs: 89 active+clean; 551 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-08-17T17:20:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3389382634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:20:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:40.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:40.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 22335 22319 199.198 151.812 0.00288164 0.00501461 2024-08-17T17:20:40.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:41.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:41.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:41.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:41.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:41.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:41.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:41.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:41.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:41.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:41.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:41.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.784 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:41.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:41.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 24984 24968 194.993 165.562 0.00413232 0.00512269 2024-08-17T17:20:41.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:41.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:42.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:42.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:42.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:42.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:42.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:42.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:42.347 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:42.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:42.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:42 smithi175 ceph-mon[101434]: pgmap v305: 89 pgs: 89 active+clean; 965 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-17T17:20:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:42 smithi064 ceph-mon[104516]: pgmap v305: 89 pgs: 89 active+clean; 965 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-17T17:20:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:42 smithi064 ceph-mon[109246]: pgmap v305: 89 pgs: 89 active+clean; 965 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-17T17:20:42.449 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:42.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:42.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:42.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:42.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:42.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:42.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:42.881 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 27513 27497 190.889 158.062 0.00789217 0.00523331 2024-08-17T17:20:42.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:42.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:43.010 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:43.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:43.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:43.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:43.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:43.423 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:43.423 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:43.427 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:43.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:43.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:43.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:43.653 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:43.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:43.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:43.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:43.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 29771 29755 185.913 141.125 0.0048646 0.00537356 2024-08-17T17:20:43.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:43.961 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:44.062 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:44.066 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:43] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:20:44.095 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:44.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:44.228 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:44.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:44.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:44.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:44 smithi175 ceph-mon[101434]: pgmap v306: 89 pgs: 89 active+clean; 1.3 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-08-17T17:20:44.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:44 smithi064 ceph-mon[104516]: pgmap v306: 89 pgs: 89 active+clean; 1.3 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-08-17T17:20:44.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:44 smithi064 ceph-mon[109246]: pgmap v306: 89 pgs: 89 active+clean; 1.3 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-08-17T17:20:44.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:44.505 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:44.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:44.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:44.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:44.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:44.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:44.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 31812 31796 180.607 127.562 0.00313907 0.00547845 2024-08-17T17:20:44.904 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:45.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:45.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:45.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:45.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:45.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:45.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:45.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:45.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:45.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.602 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:45.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:45.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:45.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.737 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:45.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:45.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:45.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:45.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:45.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 33561 33545 174.666 109.312 0.0171205 0.0057132 2024-08-17T17:20:45.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.018 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.156 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:20:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:20:46.188 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:20:46.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:46.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:46.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:46 smithi175 ceph-mon[101434]: pgmap v307: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:20:46.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:46 smithi064 ceph-mon[104516]: pgmap v307: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:20:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:46 smithi064 ceph-mon[109246]: pgmap v307: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:20:46.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:46.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:46.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:46.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:46.804 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:46.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:46.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 15 35873 35858 172.35 144.562 0.00692829 0.00579722 2024-08-17T17:20:46.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:46.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:47.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:47.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:47.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:47.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:47.351 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:47.386 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:47.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:47.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:47.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:20:47.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:47.661 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:47.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:47.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:47.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 38173 38157 170.298 143.688 0.0042079 0.0058669 2024-08-17T17:20:47.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:47.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:48.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:48.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:48.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:48.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:48.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:48.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:48.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:48 smithi175 ceph-mon[101434]: pgmap v308: 89 pgs: 89 active+clean; 2.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:20:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:48 smithi064 ceph-mon[104516]: pgmap v308: 89 pgs: 89 active+clean; 2.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:20:48.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:48 smithi064 ceph-mon[109246]: pgmap v308: 89 pgs: 89 active+clean; 2.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:20:48.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:48.505 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:48.544 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:48.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-17T17:20:48.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:48.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:48.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:48.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:48.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 40243 40227 167.57 129.375 0.00557324 0.00595831 2024-08-17T17:20:48.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:48.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:49.019 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:49.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:49.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:49.109 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:49.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:49.160 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:20:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=cleanup t=2024-08-17T17:20:48.906021053Z level=info msg="Completed cleanup jobs" duration=18.91665ms 2024-08-17T17:20:49.160 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:20:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugins.update.checker t=2024-08-17T17:20:49.054898774Z level=info msg="Update check succeeded" duration=62.954181ms 2024-08-17T17:20:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:20:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana.update.checker t=2024-08-17T17:20:49.057837773Z level=info msg="Update check succeeded" duration=91.683727ms 2024-08-17T17:20:49.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:49.197 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:49.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:49.325 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:49.427 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.461 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:49.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:49.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:49.597 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:49.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:49.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:49.839 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:49.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:49.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 42222 42206 164.827 123.688 0.00377142 0.00606153 2024-08-17T17:20:49.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:50.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:50.092 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:20:50.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:50.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:50.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:50.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:50.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:50.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:50.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[104516]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-17T17:20:50.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2805780006' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.295 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[109246]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-17T17:20:50.295 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2805780006' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.295 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:50.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:50 smithi175 ceph-mon[101434]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-17T17:20:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2805780006' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:50.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:50.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:50.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:50.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:50.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:50.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:50.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:50.882 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 44394 44378 163.116 135.75 0.00865132 0.00612612 2024-08-17T17:20:50.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:50.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:51.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:51.113 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:51.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:51.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:51.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:51.394 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:51.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:51 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:51.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:51.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:51 smithi175 ceph-mon[101434]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:20:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[104516]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:20:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:51 smithi064 ceph-mon[109246]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:20:51.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:51.530 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:51.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:51.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.809 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:51.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:51.883 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 46437 46421 161.147 127.688 0.00614759 0.00620014 2024-08-17T17:20:51.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:51.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:52.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.088 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.223 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:52.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:52.359 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:52.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:52 smithi175 ceph-mon[101434]: pgmap v311: 89 pgs: 89 active+clean; 2.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:20:52.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:52 smithi064 ceph-mon[104516]: pgmap v311: 89 pgs: 89 active+clean; 2.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:20:52.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:52 smithi064 ceph-mon[109246]: pgmap v311: 89 pgs: 89 active+clean; 2.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:20:52.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:52.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:52.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.772 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:52.883 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 48487 48471 159.409 128.125 0.00412651 0.00626819 2024-08-17T17:20:52.909 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:53.010 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:53.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:53.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:53.187 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:53.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:53.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:53.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:53.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:53.584 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:53.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.720 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.883 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:20:53.883325+0000 min lat: 0.00111021 max lat: 0.195105 avg lat: 0.00636163 2024-08-17T17:20:53.883 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:20:53.883 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 50293 50277 157.082 112.875 0.0085454 0.00636163 2024-08-17T17:20:53.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:53.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:54.069 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:20:54.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:54.070 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:20:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:20:53] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:20:54.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:54.103 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:20:54.104 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:54.135 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:20:54.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:54.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:54.169 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:20:54.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:54.207 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:20:54.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:54.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:54.244 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:20:54.245 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:54.287 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:20:54.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:54.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:54.327 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:20:54.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:54.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:54 smithi064 ceph-mon[104516]: pgmap v312: 89 pgs: 89 active+clean; 2.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-17T17:20:54.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3405952691' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:54 smithi064 ceph-mon[109246]: pgmap v312: 89 pgs: 89 active+clean; 2.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-17T17:20:54.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3405952691' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.357 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:54.357 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:20:54.358 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:20:54.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:54.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:54 smithi175 ceph-mon[101434]: pgmap v312: 89 pgs: 89 active+clean; 2.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-17T17:20:54.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3405952691' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:54.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:54.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:54.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:54.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:20:54.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:54.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:54.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:54.887 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 51916 51900 154.426 101.438 0.00629005 0.00647087 2024-08-17T17:20:54.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:55.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:55.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:55.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:55.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:55.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:55.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:55.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:55.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:55.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:55.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:55.884 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 53644 53628 152.315 108 0.00776018 0.00655995 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:53646 5.0 e115/111 5:041d0b98:::benchmark_data_smithi064_147470_object53645:head v 115'3326, mlcod=115'3322)", 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:55.887801+0000", 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0034460799999999998, 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034733490000000001, 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:20:55.893 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887801+0000", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887801+0000", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887802+0000", 2024-08-17T17:20:55.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9599999999999999e-07 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887824+0000", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1559000000000001e-05 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887838+0000", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.431e-05 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.887845+0000", 2024-08-17T17:20:55.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0090000000000001e-06 2024-08-17T17:20:55.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.888584+0000", 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073928700000000004 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.888604+0000", 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9394000000000001e-05 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:55.891154+0000", 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025508250000000001 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:20:55.898 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2024-08-17T17:20:55.899 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:20:55.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:56.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:56.108 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:56.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.246 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:56.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:56.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:56.381 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:56.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:56 smithi064 ceph-mon[104516]: pgmap v313: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 137 MiB/s wr, 2.18k op/s 2024-08-17T17:20:56.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:56 smithi064 ceph-mon[109246]: pgmap v313: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 137 MiB/s wr, 2.18k op/s 2024-08-17T17:20:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:56 smithi175 ceph-mon[101434]: pgmap v313: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 137 MiB/s wr, 2.18k op/s 2024-08-17T17:20:56.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:56.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:56.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:56.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:56.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:56.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:56.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:56.884 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 55390 55374 150.437 109.125 0.00346376 0.00664267 2024-08-17T17:20:56.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:56.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:57.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:57.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:57.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:57.203 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:57.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:57.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:57.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:57.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:57.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:57.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:20:57.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:57.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:57.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:57.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:57.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:57.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:57.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:57.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:57.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:57.884 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 56893 56877 148.082 93.9375 0.0131715 0.00674817 2024-08-17T17:20:57.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:58.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:58.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:58.160 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:58.198 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:58.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-08-17T17:20:58.221 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:58.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:58.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:20:58 smithi064 ceph-mon[104516]: pgmap v314: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-17T17:20:58.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:20:58 smithi064 ceph-mon[109246]: pgmap v314: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-17T17:20:58.390 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:58.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:20:58 smithi175 ceph-mon[101434]: pgmap v314: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-17T17:20:58.427 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:58.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:58.589 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:58.691 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:58.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:58.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:58.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:58.829 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:58.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:58.884 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 15 58555 58540 146.316 103.938 0.00892301 0.00682925 2024-08-17T17:20:58.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:59.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:59.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:59.140 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:59.242 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:59.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:59.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:59.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:59.512 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.539 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.542 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:59.641 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:59.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:59.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:59.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:20:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:59.884 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 60201 60185 144.643 102.812 0.0106162 0.006908 2024-08-17T17:20:59.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.944 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:20:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:00.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:00.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:00.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:00.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:00.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:00.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:00.363 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:00.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:00 smithi175 ceph-mon[101434]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:21:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:00 smithi064 ceph-mon[104516]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:21:00.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:00 smithi064 ceph-mon[109246]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:21:00.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:00.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:00.565 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:00.565 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:00.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:00.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:00.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:00.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:00.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:00.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:00.886 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 61854 61838 143.103 103.312 0.00661359 0.00698251 2024-08-17T17:21:00.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:00.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:01.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:01.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:01.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:01.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:01.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:01.428 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:01.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:01.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:01.593 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:01.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:01.742 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:01.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:01.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:01.886 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 63507 63491 141.682 103.312 0.0114745 0.00705362 2024-08-17T17:21:01.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:02.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:02.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:02.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:02.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:02.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:02.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:02 smithi064 ceph-mon[104516]: pgmap v316: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:21:02.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:02 smithi064 ceph-mon[109246]: pgmap v316: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:21:02.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:02.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:02.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:02 smithi175 ceph-mon[101434]: pgmap v316: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:21:02.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:02.440 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:02.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:02.542 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:02.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:02.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:02.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:02.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:02.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:02.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:02.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:02.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:02.886 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 65022 65006 140.061 94.6875 0.00691019 0.00713485 2024-08-17T17:21:02.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:02.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:02.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:03.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:03.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:03.160 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:21:03.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:03.241 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:03.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:03.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:03.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:03.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:03.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:03.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:03.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:03.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:03.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:03.761 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:03.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:03.886 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 66385 66369 138.232 85.1875 0.00665244 0.0072288 2024-08-17T17:21:03.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:03.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:03.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:04.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:04.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:04.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: pgmap v317: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:21:04.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:04.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:21:04.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:04.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:03] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: pgmap v317: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:04.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:21:04.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:04.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:21:04.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:04.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.229 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:04.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: pgmap v317: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1066684659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:04.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:04.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:04.616 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:04.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:04.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:04.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:04.887 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 67823 67807 136.672 89.875 0.00833118 0.00731072 2024-08-17T17:21:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:04.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:05.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:05.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:05.087 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:21:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:21:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:21:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:21:05.088 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:21:05.148 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:05.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:05.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:05.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:05.447 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:05.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:05.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:05.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:05.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:05.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:05.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:05.855 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.886 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 69321 69305 135.327 93.625 0.0123558 0.00738408 2024-08-17T17:21:05.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:05.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:06.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:06.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:06.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:06 smithi175 ceph-mon[101434]: pgmap v319: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:21:06.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:06 smithi064 ceph-mon[104516]: pgmap v319: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:21:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:06 smithi064 ceph-mon[109246]: pgmap v319: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:21:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:06.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:06.616 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:06.717 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:06.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:06.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.888 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 70777 70761 133.981 91 0.00900393 0.00745857 2024-08-17T17:21:06.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:07.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:07.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:07.132 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:07.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:07.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:07.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.446 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:07.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:07.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:07.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:07.718 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:07.820 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:07.887 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 72213 72197 132.68 89.75 0.00590691 0.00753217 2024-08-17T17:21:07.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:08.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:08.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.139 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:08.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:08.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:08.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:08.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:08 smithi175 ceph-mon[101434]: pgmap v320: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:21:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:08 smithi064 ceph-mon[104516]: pgmap v320: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:21:08.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:08 smithi064 ceph-mon[109246]: pgmap v320: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:21:08.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:08.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:08.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:08.670 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:08.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:08.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:08.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:08.888 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 15 73590 73575 131.35 86.125 0.00626905 0.0076083 2024-08-17T17:21:08.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:08.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:09.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:09.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:09.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:09.193 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:09.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:09.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.330 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:09.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:09.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:09.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:09.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:21:09.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:09.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:09.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:09.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:09.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:09.889 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 74860 74844 129.898 79.3125 0.00901411 0.00769252 2024-08-17T17:21:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:09.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:10.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.045 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:10.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:10.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:10.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:10.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:10.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[104516]: pgmap v321: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[109246]: pgmap v321: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:10.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:10.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:10.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:10.514 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:10 smithi175 ceph-mon[101434]: pgmap v321: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:21:10.514 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:10.514 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:10.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:10.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:10.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:10.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:10.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:10.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:10.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:10.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 76067 76051 128.427 75.4375 0.0116655 0.00778183 2024-08-17T17:21:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:10.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:11.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/589065628' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/589065628' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:11.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:11.254 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:21:11.254 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:11.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:11.291 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:11.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:21:11.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:11.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:11.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:21:11.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:11.368 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:21:11.368 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:21:11.368 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:21:11.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:21:11.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:21:11.378 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:11.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.414 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:21:11.414 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:11.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:11.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:11.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:11.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:21:11.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:11 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/589065628' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:11.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:11.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:11.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:11.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 77168 77152 126.858 68.8125 0.00782452 0.00787725 2024-08-17T17:21:11.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:12.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:12.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:12.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:12.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:12.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:12.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:12.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:12 smithi064 ceph-mon[104516]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:21:12.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:12 smithi064 ceph-mon[109246]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:21:12.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:12.557 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:12.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:12 smithi175 ceph-mon[101434]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:21:12.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:12.796 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:12.828 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:12.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 78612 78596 125.919 90.25 0.00958067 0.00793664 2024-08-17T17:21:12.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:13.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:13.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:13.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:13.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:21:13.282 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:21:13.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-08-17T17:21:13.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:13.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:13.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:13.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:13.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:13.653 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:13.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.790 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:13.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:13.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:21:13.890220+0000 min lat: 0.00111021 max lat: 0.195105 avg lat: 0.00800027 2024-08-17T17:21:13.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:21:13.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 79985 79969 124.917 85.8125 0.0207306 0.00800027 2024-08-17T17:21:13.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:13.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:13.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:14.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:14.070 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:13] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:21:14.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:14.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:14.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:14.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.368 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:14.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:14 smithi064 ceph-mon[104516]: pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:21:14.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:14 smithi064 ceph-mon[109246]: pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:21:14.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:14.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:14.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:14.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:14.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:14 smithi175 ceph-mon[101434]: pgmap v323: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:21:14.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:14.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:14.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:14.889 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:14.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 81361 81345 123.967 86 0.0184259 0.00806065 2024-08-17T17:21:14.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:14.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:15.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:15.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:15.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:15.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:15.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:15.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:15.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:15.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:15.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:15.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:15.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:15.704 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:15.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:15.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:15.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:15.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 82607 82591 122.869 77.875 0.00524299 0.00813413 2024-08-17T17:21:15.938 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:15.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:16.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.118 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:16.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:16.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:16.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:16.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:16 smithi175 ceph-mon[101434]: pgmap v324: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:21:16.666 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:16.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:16 smithi064 ceph-mon[104516]: pgmap v324: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:21:16.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:16 smithi064 ceph-mon[109246]: pgmap v324: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:21:16.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:16.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.800 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:16.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:16.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 83922 83906 121.923 82.1875 0.0108607 0.0081967 2024-08-17T17:21:16.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:16.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:16.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:17.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:17.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:17.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:17.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:17.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:17.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:17.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:17.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:17.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:17.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:17.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:17.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:17.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:17.761 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:17.761 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:17.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:17.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:17.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 85254 85238 121.044 83.25 0.0123978 0.00825621 2024-08-17T17:21:17.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.909 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:17.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:18.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:18.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:18.151 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:18.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:18.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:18.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:18 smithi064 ceph-mon[104516]: pgmap v325: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:21:18.319 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:18 smithi064 ceph-mon[109246]: pgmap v325: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:21:18.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:18.427 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:18.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:18.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:18.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:18.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:18 smithi175 ceph-mon[101434]: pgmap v325: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:21:18.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:18.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:18.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:18.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:18.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:18.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 86534 86518 120.132 80 0.0126355 0.00831956 2024-08-17T17:21:18.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:18.998 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:19.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:19.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:19.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:19.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:19.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:19.375 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:19.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:19.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:21:19.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:19.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:19.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:19.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:19.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:19.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:19.890 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 87849 87833 119.307 82.1875 0.00502948 0.00837682 2024-08-17T17:21:19.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:19.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:19.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:20.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:20.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.321 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:20 smithi064 ceph-mon[104516]: pgmap v326: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:20.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:20 smithi064 ceph-mon[109246]: pgmap v326: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:20.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.386 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.449 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:20.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.546 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:20.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:21:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:21:20.554 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:21:20.554 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:21:20.647 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:20 smithi175 ceph-mon[101434]: pgmap v326: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:20.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:20.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:20.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 89057 89041 118.375 75.5 0.0152989 0.00844271 2024-08-17T17:21:20.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:20.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:21.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:21.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:21.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:21.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:21.343 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:21.382 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:21.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:21.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:21:21.594 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:21:21.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-08-17T17:21:21.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:21.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:21.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:21.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:21.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 90254 90238 117.468 74.8125 0.012131 0.00850799 2024-08-17T17:21:21.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:21.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.031 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:22.064 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:22.165 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:22.346 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:21:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:22.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[104516]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:21:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/134518910' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[109246]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:21:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/134518910' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.449 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:22.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:22.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:22.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:22 smithi175 ceph-mon[101434]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:21:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:22 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/134518910' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:22.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 91450 91434 116.596 74.75 0.0204025 0.00857127 2024-08-17T17:21:22.892 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.911 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:22.994 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:23.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:23.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:23.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:23.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.307 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:23.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[104516]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:23.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-mon[109246]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:21:23.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:23.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:23.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:23.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:23 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:23 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:23 smithi175 ceph-mon[101434]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:21:23.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:23.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:23.886 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:23.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 92604 92588 115.706 72.125 0.00830817 0.00863814 2024-08-17T17:21:23.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:24.165 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:24.213 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:23] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:21:24.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:24.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.524 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:24 smithi064 ceph-mon[104516]: pgmap v329: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-08-17T17:21:24.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:24 smithi064 ceph-mon[109246]: pgmap v329: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-08-17T17:21:24.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:24.592 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:24 smithi175 ceph-mon[101434]: pgmap v329: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-08-17T17:21:24.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:24.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.829 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:24.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:24.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 93817 93801 114.924 75.8125 0.0210642 0.00869651 2024-08-17T17:21:24.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:24.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:25.099 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:25.129 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:25.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:25.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:25.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:25.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:25.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:25.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:25.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:25.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:25.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:25.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:25.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 95026 95010 114.167 75.5625 0.00818181 0.00875412 2024-08-17T17:21:25.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:25.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:26.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:26.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:26.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:21:26.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:26.235 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:26.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[104516]: pgmap v330: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:21:26.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:26.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:26.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[109246]: pgmap v330: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:21:26.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:26.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:26.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:26.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:26 smithi175 ceph-mon[101434]: pgmap v330: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:21:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:26.465 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:21:26.465 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:26.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:26.498 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:21:26.498 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:26.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:26.535 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:21:26.535 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:26.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:21:26.567 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:26.605 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:21:26.606 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:26.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:26.639 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:21:26.639 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:26.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:26.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:21:26.671 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:26.703 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:21:26.703 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:26.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:26.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:26.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:26.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 96082 96066 113.258 66 0.0264803 0.00882256 2024-08-17T17:21:26.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:26.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:27.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:27.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:27.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1418293031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:27.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1418293031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:27.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.333 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:27.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:27.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1418293031' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:27.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:27.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:27.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:27.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:27.610 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:27.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:27.617 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:27.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:27.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:27.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 97113 97097 112.354 64.4375 0.012662 0.00889529 2024-08-17T17:21:27.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.016 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:28.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:28.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:28 smithi064 ceph-mon[104516]: pgmap v331: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:28.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:28 smithi064 ceph-mon[109246]: pgmap v331: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:28.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.289 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:28.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:28.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:28 smithi175 ceph-mon[101434]: pgmap v331: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:21:28.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:28.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:28.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:28.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:28.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 98254 98238 111.608 71.3125 0.0162239 0.00895462 2024-08-17T17:21:28.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:28.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:29.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:29.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:29.217 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:29.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:29.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:29.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:29.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:29.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:29.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:29.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:29.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:29.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:29.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:29.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 99416 99400 110.912 72.625 0.00796193 0.00901119 2024-08-17T17:21:29.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:29.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:30.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:30.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:30.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.259 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:30.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:30.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:30.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:30 smithi175 ceph-mon[101434]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:21:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:30 smithi064 ceph-mon[104516]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:21:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:30 smithi064 ceph-mon[109246]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:21:30.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:30.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:30.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:30.665 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:30.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:30.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:30.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:30.891 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 100575 100559 110.237 72.4375 0.0147292 0.00906629 2024-08-17T17:21:30.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:30.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:30.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:31.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.090 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:31.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:31.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:31.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:31.332 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.359 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.461 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:31.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:31.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:31.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:31.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:31.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 101709 101693 109.558 70.875 0.0110391 0.0091219 2024-08-17T17:21:31.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:32.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:32.196 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:32.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:32 smithi064 ceph-mon[104516]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:21:32.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:32 smithi064 ceph-mon[109246]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:21:32.297 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:32.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:32.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:32 smithi175 ceph-mon[101434]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:21:32.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:32.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:32.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.604 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:102450 5.b 5:de7aa858:::benchmark_data_smithi064_147470_object102449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:32.582318+0000", 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.025265105999999999, 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025299815, 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:32.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102450 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582318+0000", 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582318+0000", 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582320+0000", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.511e-06 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582345+0000", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5979e-05 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582363+0000", 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7816999999999999e-05 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582372+0000", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7900000000000005e-06 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582784+0000", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041236199999999999 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.582930+0000", 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014566199999999999 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.583064+0000", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133926 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.589834+0000", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067700080000000001 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.590809+0000", 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097457999999999996 2024-08-17T17:21:32.614 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:102471 5.a 5:5c739b76:::benchmark_data_smithi064_147470_object102470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:32.603430+0000", 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0041529430000000001, 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00426728, 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:32.615 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102471 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603430+0000", 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603430+0000", 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603432+0000", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.902e-06 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603478+0000", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5466999999999997e-05 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603498+0000", 2024-08-17T17:21:32.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0044999999999999e-05 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603505+0000", 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4100000000000002e-06 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603569+0000", 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3862000000000006e-05 2024-08-17T17:21:32.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603712+0000", 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143566 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603863+0000", 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150626 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.619 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:102473 5.8 5:1d13081e:::benchmark_data_smithi064_147470_object102472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:32.603525+0000", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.004057817, 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042347519999999996, 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102473 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:32.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603525+0000", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603525+0000", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603526+0000", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7199999999999999e-07 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:32.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603549+0000", 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.349e-05 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603557+0000", 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1270000000000003e-06 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.603563+0000", 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5439999999999998e-06 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604267+0000", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070403600000000001 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604430+0000", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016268200000000001 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604554+0000", 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012404199999999999 2024-08-17T17:21:32.623 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:102477 5.6 5:6dbeb268:::benchmark_data_smithi064_147470_object102476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:32.604112+0000", 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0034707929999999998, 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003709127, 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:32.624 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102477 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604112+0000", 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604112+0000", 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:32.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604113+0000", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3799999999999996e-07 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604161+0000", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7474e-05 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604172+0000", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1764e-05 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:32.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604179+0000", 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6599999999999998e-06 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.605822+0000", 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016432700000000001 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.605950+0000", 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012736799999999999 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.606069+0000", 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011957500000000001 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:102478 5.6 5.c503ff06 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:32.604197+0000", 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.003386455, 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036806120000000002, 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:32.628 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102478 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604197+0000", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604197+0000", 2024-08-17T17:21:32.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604197+0000", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5400000000000001e-07 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604219+0000", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2164e-05 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604229+0000", 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0037000000000001e-05 2024-08-17T17:21:32.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:32.604235+0000", 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1889999999999998e-06 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 5 2024-08-17T17:21:32.631 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:21:32.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.843 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:32.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:32.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 102800 102784 108.857 68.1875 0.0255466 0.00918092 2024-08-17T17:21:32.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:33.008 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:33.067 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:21:33.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2343351378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2343351378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:33.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:33.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:33.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.288 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:33.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:33.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2343351378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:33.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:33.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:33.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:33.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:33.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:33.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:33.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:33.844 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:33.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:21:33.892308+0000 min lat: 0.00111021 max lat: 0.195105 avg lat: 0.00923586 2024-08-17T17:21:33.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:21:33.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 103923 103907 108.213 70.1875 0.0151837 0.00923586 2024-08-17T17:21:33.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:33.967 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:33] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-08-17T17:21:33.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:21:33.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:34.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:34.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[104516]: pgmap v334: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:21:34.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:34.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:34.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[104516]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:21:34.268 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[109246]: pgmap v334: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:21:34.268 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:34.268 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:34.268 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:34 smithi064 ceph-mon[109246]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:21:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.269 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:34.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:21:34.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:21:34.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:34.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:34 smithi175 ceph-mon[101434]: pgmap v334: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:21:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:34 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:34 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:34 smithi175 ceph-mon[101434]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:21:34.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:34.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:34.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:34.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:34.879 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:34.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 105012 104996 107.554 68.0625 0.0136924 0.0092919 2024-08-17T17:21:34.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:34.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:35.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:35.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:35.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:35.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:35.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:35.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:35.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:35.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:35.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:35.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:35.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:35.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:35.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 105946 105930 106.761 58.375 0.071702 0.00935952 2024-08-17T17:21:35.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:35.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:36.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:36.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: pgmap v336: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:21:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: pgmap v336: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:21:36.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:36.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:36.347 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: pgmap v336: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:21:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:21:36.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:36.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:36.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:36.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:36.761 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:36.863 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:36.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 107069 107053 106.18 70.1875 0.00737904 0.00941246 2024-08-17T17:21:36.895 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:36.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:37.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:37.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:37.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:37.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:37.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:37.255 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:37.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:37.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:21:37.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:37.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:37.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:37.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:37.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:37.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:37.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:37.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 108174 108158 105.6 69.0625 0.00567748 0.0094648 2024-08-17T17:21:37.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:38.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:38.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:38.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:37 smithi175 ceph-mon[101434]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:37 smithi064 ceph-mon[104516]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:38.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:37 smithi064 ceph-mon[109246]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:38.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:38.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:38.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:21:38.325 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:21:38.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-17T17:21:38.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:38.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:38.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:38.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:38.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:38.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:38.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.691 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.793 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:38.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 109251 109235 105.011 67.3125 0.00634687 0.00951804 2024-08-17T17:21:38.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:38.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:39.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:39.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:39.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:39.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:39.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:39.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:39.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:39.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:39.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:39.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:39.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 110349 110333 104.46 68.625 0.0137732 0.00956753 2024-08-17T17:21:39.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:39.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.032 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:39 smithi064 ceph-mon[104516]: pgmap v338: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:40.032 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:39 smithi064 ceph-mon[109246]: pgmap v338: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:40.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:40.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:40.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.260 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:40.266 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:40.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:40.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:39 smithi175 ceph-mon[101434]: pgmap v338: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:21:40.491 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:40.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:40.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:40.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:40.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 111343 111327 103.828 62.125 0.0230394 0.00962619 2024-08-17T17:21:40.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:40.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:41.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:41.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:41.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:41.118 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:21:41.119 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:41.156 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:21:41.156 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:41.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:41.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:41.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:41.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:21:41.204 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:41.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:41.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:41.246 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:21:41.246 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:41.282 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:21:41.282 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:41.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:41.318 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:21:41.318 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:41.351 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:21:41.351 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:41.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:41.389 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:21:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:41.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:41.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:41.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:41.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:41.648 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:41.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:41.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:41.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:41.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 112318 112302 103.197 60.9375 0.00749008 0.00968489 2024-08-17T17:21:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:41.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:42.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:42.054 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:42.080 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:41 smithi064 ceph-mon[104516]: pgmap v339: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:21:42.080 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3562276446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:42.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:41 smithi064 ceph-mon[109246]: pgmap v339: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:21:42.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3562276446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:42.156 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:42.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:42.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:42.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:41 smithi175 ceph-mon[101434]: pgmap v339: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:21:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3562276446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:42.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:42.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:42.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:42.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:42.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:42.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:42.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:42.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:42.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 113370 113354 102.654 65.75 0.0234429 0.00973641 2024-08-17T17:21:42.989 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:43.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:43.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:43.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:43.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:43.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:43.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:43.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.538 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:43.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:43.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:43.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:43.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 114461 114445 102.162 68.1875 0.00947872 0.00978358 2024-08-17T17:21:43.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:43.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:44.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:44.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:43 smithi064 ceph-mon[104516]: pgmap v340: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:21:44.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:43 smithi064 ceph-mon[109246]: pgmap v340: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:21:44.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:43] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:21:44.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:44.265 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:44.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:44.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:44.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:43 smithi175 ceph-mon[101434]: pgmap v340: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:21:44.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:44.545 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:44.647 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:44.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:44.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:44.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:44.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 115547 115531 101.679 67.875 0.0271402 0.00983032 2024-08-17T17:21:44.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:44.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:45.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.182 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:45.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:45.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.319 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:45.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:45.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:45.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:45.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:45.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:45.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:45.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:45.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:45.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:45.893 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 116564 116548 101.149 63.5625 0.0264648 0.00988106 2024-08-17T17:21:45.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:46.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:46.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:46.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:45 smithi064 ceph-mon[104516]: pgmap v341: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:21:46.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:45 smithi064 ceph-mon[109246]: pgmap v341: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:21:46.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:46.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:46.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:46.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:46.289 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:46.289 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:0 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:1 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:2 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:bench [] [] [] [] 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:21:46.336 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2024-08-17T17:21:46.337 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2024-08-17T17:21:46.337 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore allocator dump block 2024-08-17T17:21:46.337 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: invalid command 2024-08-17T17:21:46.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:45 smithi175 ceph-mon[101434]: pgmap v341: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:21:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.422 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:46.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:46.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:46.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:46.659 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:46.794 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:46.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:46.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 117541 117525 100.6 61.0625 0.0167804 0.00993535 2024-08-17T17:21:46.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:46.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:47.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:47.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:47.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1532989731' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1532989731' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:21:47.213 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:21:47.248 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:47.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.381 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:47.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1532989731' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:47.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:47.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:47.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:47.659 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:47.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:47.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:47.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 118562 118546 100.103 63.8125 0.0133738 0.00998494 2024-08-17T17:21:47.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:47.935 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:48.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:48.062 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:48.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:48.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:48.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:48.223 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:21:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[104516]: pgmap v342: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:21:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[104516]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:21:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[109246]: pgmap v342: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:21:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:47 smithi064 ceph-mon[109246]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:21:48.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.338 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:48.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:47 smithi175 ceph-mon[101434]: pgmap v342: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:21:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:47 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:47 smithi175 ceph-mon[101434]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:21:48.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:48.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:48.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:48.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:48.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:48.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:48.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 119533 119517 99.5774 60.6875 0.0188921 0.0100371 2024-08-17T17:21:48.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:49.023 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:49.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:49.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:49.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:49.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:49.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:49.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:49.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:49.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:49.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:49.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:49.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:49.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:49.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:21:49.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:49.873 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:49.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 120567 120551 99.1175 64.625 0.0120961 0.0100845 2024-08-17T17:21:49.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:50.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:49 smithi064 ceph-mon[104516]: pgmap v344: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:21:50.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:49 smithi064 ceph-mon[109246]: pgmap v344: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:21:50.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.162 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:50.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:50.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:50.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:50.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:50.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:49 smithi175 ceph-mon[101434]: pgmap v344: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:21:50.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:50.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:50.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:50.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:50.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:50.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 121590 121574 98.6606 63.9375 0.0168988 0.0101311 2024-08-17T17:21:50.977 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:51.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:51.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:51.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:51.175 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:51.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:51.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:51.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:51.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:51.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:21:51.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:51.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:51.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:51.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:51.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:51.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:51.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 122569 122553 98.1802 61.1875 0.00706186 0.0101809 2024-08-17T17:21:51.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:52.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:52.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:52.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:51 smithi064 ceph-mon[104516]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:21:52.154 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:51 smithi064 ceph-mon[109246]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:21:52.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:52.213 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:52.214 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:21:52.214 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:52.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd out 4 2024-08-17T17:21:52.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:52.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:52.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:52.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:51 smithi175 ceph-mon[101434]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:21:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:52.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:52.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:21:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:52.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:52.565 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:21:52.566 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:52.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:52.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:52.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:21:52.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:52.636 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:21:52.636 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:52.666 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:21:52.666 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:52.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:52.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:21:52.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:52.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.717 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:52.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:52.740 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:21:52.740 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:52.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:21:52.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:52.818 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:52.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:52.894 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 123529 123513 97.6968 60 0.00953309 0.0102313 2024-08-17T17:21:52.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:53.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:53.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1070415338' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:53.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1070415338' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:53.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:53.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:53.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1070415338' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.426 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:53.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:53.527 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:53.557 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:53.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:53.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.701 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:53.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:53.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:53.808 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:53.855 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:53.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:53.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:21:53.895313+0000 min lat: 0.00111021 max lat: 0.195105 avg lat: 0.0102818 2024-08-17T17:21:53.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:21:53.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 124468 124452 97.2084 58.6875 0.0302816 0.0102818 2024-08-17T17:21:53.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:54.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:54.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:54.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:54.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:54.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:54.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:53 smithi064 ceph-mon[104516]: pgmap v346: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:54.202 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:21:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:21:53] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:21:54.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:53 smithi064 ceph-mon[109246]: pgmap v346: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:54.244 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:54.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:54.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:54.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:53 smithi175 ceph-mon[101434]: pgmap v346: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:54.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:54.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:54.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:54.656 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:54.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:54.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:54.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:54.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:54.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 125311 125295 96.6587 52.6875 0.010496 0.0103404 2024-08-17T17:21:54.935 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:54.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:55.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.099 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:55.234 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:55.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.472 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:55.505 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:55.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:55.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:55.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:55.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:55.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 126263 126247 96.2056 59.5 0.00798841 0.0103894 2024-08-17T17:21:55.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:56.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:56.127 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:55 smithi064 ceph-mon[104516]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:55 smithi064 ceph-mon[109246]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:56.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:56.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:56.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:55 smithi175 ceph-mon[101434]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:21:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:56.535 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:56.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:56.637 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:56.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.676 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:56.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:56.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:56.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:56.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 127177 127161 95.7347 57.125 0.0348437 0.0104403 2024-08-17T17:21:56.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:56.944 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:57.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:57.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:57.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:57.220 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:57.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:57.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:57.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:57.459 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:57.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:57.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:57.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:57.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:57.764 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:57.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:57.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 128044 128028 95.2401 54.1875 0.014437 0.0104949 2024-08-17T17:21:57.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:57.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:58.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:58.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:58.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:58.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:58 smithi064 ceph-mon[104516]: pgmap v348: 89 pgs: 89 active+clean; 7.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-08-17T17:21:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:58 smithi064 ceph-mon[109246]: pgmap v348: 89 pgs: 89 active+clean; 7.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-08-17T17:21:58.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:58.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:58.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:58.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:58 smithi175 ceph-mon[101434]: pgmap v348: 89 pgs: 89 active+clean; 7.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-08-17T17:21:58.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:58.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:58.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:15418 5.d 5:b29f855f:::benchmark_data_smithi064_147470_object15417:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.288384+0000", 2024-08-17T17:21:58.522 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.228231417999993, 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.105005872, 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 15418 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.523 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288384+0000", 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288384+0000", 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288385+0000", 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156e-06 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288408+0000", 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3264999999999998e-05 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288419+0000", 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0910999999999999e-05 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288427+0000", 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288559+0000", 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013246699999999999 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288674+0000", 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115081 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.288822+0000", 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014752100000000001 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289738+0000", 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000916679 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290978+0000", 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001240115 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393277+0000", 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10229885700000001 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393340+0000", 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2277e-05 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393390+0000", 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0099999999999998e-05 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:15424 5.d 5:bbb924fc:::benchmark_data_smithi064_147470_object15423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.289842+0000", 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.226772916000002, 2024-08-17T17:21:58.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103676478, 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 15424 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289842+0000", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289842+0000", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289843+0000", 2024-08-17T17:21:58.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0419999999999999e-06 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289867+0000", 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3129000000000002e-05 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289876+0000", 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7880000000000001e-06 2024-08-17T17:21:58.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289893+0000", 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6342999999999999e-05 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.289941+0000", 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8554000000000003e-05 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290054+0000", 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011275799999999999 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290182+0000", 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127682 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290895+0000", 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071336499999999998 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392515+0000", 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101620399 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393435+0000", 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00092003199999999999 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393483+0000", 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7253999999999999e-05 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393519+0000", 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6131999999999999e-05 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:15425 5.8 5:12ae6c11:::benchmark_data_smithi064_147470_object15424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.290221+0000", 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.226393994999995, 2024-08-17T17:21:58.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103181378, 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 15425 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290221+0000", 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290221+0000", 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290222+0000", 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.33e-07 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290245+0000", 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2449000000000001e-05 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290255+0000", 2024-08-17T17:21:58.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0040999999999999e-05 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290262+0000", 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0360000000000001e-06 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290325+0000", 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3659999999999997e-05 2024-08-17T17:21:58.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290399+0000", 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3627000000000005e-05 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.290474+0000", 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4629999999999995e-05 2024-08-17T17:21:58.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291308+0000", 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00083437300000000002 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392449+0000", 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101141192 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393312+0000", 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00086319100000000002 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393363+0000", 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0575999999999998e-05 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393403+0000", 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9669999999999998e-05 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:15429 5.a 5:56a4c2e5:::benchmark_data_smithi064_147470_object15428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.291199+0000", 2024-08-17T17:21:58.547 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.225416738000007, 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10529538500000001, 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 15429 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.548 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291199+0000", 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291199+0000", 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291200+0000", 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159e-06 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291218+0000", 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7821000000000001e-05 2024-08-17T17:21:58.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291226+0000", 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.619e-06 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291231+0000", 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7169999999999999e-06 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291283+0000", 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2009000000000003e-05 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393014+0000", 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101730838 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393147+0000", 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013360800000000001 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396240+0000", 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030929730000000002 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396280+0000", 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9424000000000001e-05 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396385+0000", 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010511299999999999 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396446+0000", 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1593999999999993e-05 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396494+0000", 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7509999999999997e-05 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:15433 5.a 5:594e11f0:::benchmark_data_smithi064_147470_object15432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.291637+0000", 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.224977781000007, 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10682447, 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.556 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 15433 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291637+0000", 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291637+0000", 2024-08-17T17:21:58.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:58.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291638+0000", 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9399999999999993e-07 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291659+0000", 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0704e-05 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392972+0000", 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101313009 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392983+0000", 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0987e-05 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.394438+0000", 2024-08-17T17:21:58.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014544110000000001 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.394557+0000", 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118961 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.394686+0000", 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129054 2024-08-17T17:21:58.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.396343+0000", 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016572030000000001 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.398061+0000", 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001717779 2024-08-17T17:21:58.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.398341+0000", 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000280356 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.398397+0000", 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5837000000000001e-05 2024-08-17T17:21:58.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.398462+0000", 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5174999999999999e-05 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:15425 5.8 e114/111 ondisk, result = 0)", 2024-08-17T17:21:58.565 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.291710+0000", 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.224905383000007, 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.100747718, 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291710+0000", 2024-08-17T17:21:58.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291710+0000", 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291711+0000", 2024-08-17T17:21:58.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7100000000000001e-07 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291714+0000", 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4309999999999998e-06 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392315+0000", 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10060121 2024-08-17T17:21:58.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392334+0000", 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8683000000000001e-05 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392431+0000", 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7403000000000002e-05 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392442+0000", 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0957e-05 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.392458+0000", 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5362999999999998e-05 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:15418 5.d e114/111 ondisk, result = 0)", 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:38.291776+0000", 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.224839392999996, 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10162539800000001, 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291776+0000", 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291776+0000", 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291776+0000", 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4000000000000002e-07 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.291781+0000", 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3479999999999997e-06 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393209+0000", 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101428507 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393215+0000", 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9190000000000004e-06 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393264+0000", 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8936e-05 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393272+0000", 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:38.393401+0000", 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012961900000000001 2024-08-17T17:21:58.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:56676 5.d 5:bdcb2d39:::benchmark_data_smithi064_147470_object56675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:57.656933+0000", 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 60.859681774000002, 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11400354, 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.576 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 56676 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656933+0000", 2024-08-17T17:21:58.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656933+0000", 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656934+0000", 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7599999999999997e-07 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656957+0000", 2024-08-17T17:21:58.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2603e-05 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656966+0000", 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3700000000000001e-06 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.656972+0000", 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3799999999999999e-06 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659420+0000", 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024477349999999999 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659548+0000", 2024-08-17T17:21:58.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012832300000000001 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659687+0000", 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013851400000000001 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.662306+0000", 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026188600000000002 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.665149+0000", 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028429689999999999 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.770772+0000", 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10562276800000001 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.770875+0000", 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103674 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.770937+0000", 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1767999999999995e-05 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:56678 5.6 5:616d534c:::benchmark_data_smithi064_147470_object56677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:21:58.584 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:57.659918+0000", 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 60.856697072999999, 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11108241100000001, 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 56678 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659918+0000", 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659918+0000", 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659920+0000", 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325e-06 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659945+0000", 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5812999999999998e-05 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659958+0000", 2024-08-17T17:21:58.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2437e-05 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.659966+0000", 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2390000000000002e-06 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.660027+0000", 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0778e-05 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.660144+0000", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011678 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.660272+0000", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012798400000000001 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.662782+0000", 2024-08-17T17:21:58.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025101490000000001 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.664375+0000", 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015930230000000001 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.770855+0000", 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.106480096 2024-08-17T17:21:58.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.770960+0000", 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104889 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.771001+0000", 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0898e-05 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.591 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:56681 5.d 5:b49de1ad:::benchmark_data_smithi064_147470_object56680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:57.661845+0000", 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 60.854770016000003, 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11472138, 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 56681 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661845+0000", 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661845+0000", 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661847+0000", 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5209999999999999e-06 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661870+0000", 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2802999999999998e-05 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661881+0000", 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1100999999999999e-05 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.661895+0000", 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4817000000000001e-05 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.662506+0000", 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00061089599999999999 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.662655+0000", 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148183 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.662792+0000", 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013712300000000001 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.664559+0000", 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017675119999999999 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.665187+0000", 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062788399999999999 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.776402+0000", 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.111215023 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.776464+0000", 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1507999999999995e-05 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.776567+0000", 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103009 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:56694 5.d 5:b41a1daf:::benchmark_data_smithi064_147470_object56693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:21:58.600 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:20:57.669545+0000", 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 60.847069990999998, 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.110856066, 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 56694 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.601 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669545+0000", 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669545+0000", 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669546+0000", 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1570000000000001e-06 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669567+0000", 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.105e-05 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669578+0000", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0567999999999999e-05 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669584+0000", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3230000000000003e-06 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669624+0000", 2024-08-17T17:21:58.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9428999999999998e-05 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669720+0000", 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6416000000000007e-05 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.669812+0000", 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2050000000000001e-05 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.671307+0000", 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014949309999999999 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.671654+0000", 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034677599999999997 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.780261+0000", 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10860744 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.780360+0000", 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9075999999999998e-05 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:20:57.780401+0000", 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0849999999999997e-05 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:73627 5.b 5:dfcbf5db:::benchmark_data_smithi064_147470_object73626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:08.909945+0000", 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.606669793999998, 2024-08-17T17:21:58.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093308985999999997, 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 73627 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909945+0000", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909945+0000", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909947+0000", 2024-08-17T17:21:58.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4160000000000001e-06 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909970+0000", 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3059000000000001e-05 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909982+0000", 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1640999999999999e-05 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.909988+0000", 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7490000000000001e-06 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.910846+0000", 2024-08-17T17:21:58.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00085771899999999995 2024-08-17T17:21:58.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.910978+0000", 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013231399999999999 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.911104+0000", 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012553999999999999 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.964270+0000", 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.053165761999999998 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.966042+0000", 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001772238 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:08.979207+0000", 2024-08-17T17:21:58.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013164946 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:09.003209+0000", 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024001855999999998 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:09.003254+0000", 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5745999999999999e-05 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.616 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:95993 5.6 5:6dfc5a52:::benchmark_data_smithi064_147470_object95992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.728492+0000", 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 31.788123082999999, 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.080977968999999997, 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.617 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95993 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728492+0000", 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728492+0000", 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728494+0000", 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.826e-06 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728520+0000", 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6356999999999999e-05 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728532+0000", 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2131e-05 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728541+0000", 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5879999999999998e-06 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728616+0000", 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4578000000000003e-05 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728764+0000", 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148529 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.728944+0000", 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017959700000000001 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.732336+0000", 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033922679999999999 2024-08-17T17:21:58.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736089+0000", 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037526730000000002 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809340+0000", 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073251073999999999 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809418+0000", 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8338999999999997e-05 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809470+0000", 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2009000000000003e-05 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:95996 5.6 5:6e156f64:::benchmark_data_smithi064_147470_object95995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:58.624 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.729281+0000", 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 31.787334756, 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.080303558999999997, 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95996 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729281+0000", 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729281+0000", 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729281+0000", 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8299999999999995e-07 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729306+0000", 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4138000000000002e-05 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729319+0000", 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3542e-05 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729325+0000", 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7599999999999999e-06 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.732364+0000", 2024-08-17T17:21:58.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030391709999999998 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.732482+0000", 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117737 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.732612+0000", 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013073100000000001 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735813+0000", 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003201003 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736129+0000", 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031506399999999999 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809513+0000", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073384700999999997 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809546+0000", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2543000000000003e-05 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.809584+0000", 2024-08-17T17:21:58.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8285999999999999e-05 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:95997 5.a 5:58454645:::benchmark_data_smithi064_147470_object95996:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.729345+0000", 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 31.787269965, 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094777817, 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.632 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95997 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729345+0000", 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729345+0000", 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729346+0000", 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2300000000000001e-07 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729368+0000", 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2569e-05 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729378+0000", 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6970000000000005e-06 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729384+0000", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1589999999999998e-06 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729449+0000", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4523999999999994e-05 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729577+0000", 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012763499999999999 2024-08-17T17:21:58.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.729681+0000", 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010447500000000001 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.733138+0000", 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034575109999999999 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.733990+0000", 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00085116499999999997 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824042+0000", 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090052744000000004 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824081+0000", 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8469e-05 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824123+0000", 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2246000000000002e-05 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.639 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:96005 5.b 5:d423ca7f:::benchmark_data_smithi064_147470_object96004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.736001+0000", 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 31.780614601, 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.081314195000000006, 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.640 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96005 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736001+0000", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736001+0000", 2024-08-17T17:21:58.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736003+0000", 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9240000000000001e-06 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736025+0000", 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2515999999999999e-05 2024-08-17T17:21:58.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736035+0000", 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7119999999999995e-06 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736042+0000", 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0570000000000004e-06 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736578+0000", 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053653700000000002 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736687+0000", 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108553 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:21:58.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736814+0000", 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127491 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.740188+0000", 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003373479 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.743592+0000", 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034044769999999999 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.817185+0000", 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073592829999999998 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.817258+0000", 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2408000000000002e-05 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.817315+0000", 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7210999999999998e-05 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:96006 5.a 5:51458492:::benchmark_data_smithi064_147470_object96005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.736404+0000", 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 31.780211587, 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094017199999999995, 2024-08-17T17:21:58.647 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96006 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736404+0000", 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736404+0000", 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736405+0000", 2024-08-17T17:21:58.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4099999999999997e-07 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736425+0000", 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0733e-05 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736436+0000", 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0328e-05 2024-08-17T17:21:58.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736443+0000", 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9670000000000004e-06 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736528+0000", 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5154000000000005e-05 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736680+0000", 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015213600000000001 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.737065+0000", 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038551299999999998 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.740446+0000", 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003380405 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.740844+0000", 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039770000000000002 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.830312+0000", 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.089468844000000006 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.830375+0000", 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2436999999999999e-05 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.830421+0000", 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6041999999999997e-05 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.654 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:105896 5.b 5:d42da47a:::benchmark_data_smithi064_147470_object105895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:35.814070+0000", 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.702545575999999, 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.084688202000000004, 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 105896 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814070+0000", 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814070+0000", 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814071+0000", 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.282e-06 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814103+0000", 2024-08-17T17:21:58.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1806e-05 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814113+0000", 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0049e-05 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814119+0000", 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5989999999999998e-06 2024-08-17T17:21:58.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.814666+0000", 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054676599999999996 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.852109+0000", 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037442419999999997 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.852273+0000", 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016415700000000001 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.859419+0000", 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071462380000000001 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.859460+0000", 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0864e-05 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.898534+0000", 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039073631999999997 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.898713+0000", 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179968 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:35.898758+0000", 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4421e-05 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:124728 5.6 5:6d2bc325:::benchmark_data_smithi064_147470_object124727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:54.175755+0000", 2024-08-17T17:21:58.662 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.3408603939999999, 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082886099000000005, 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124728 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175755+0000", 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175755+0000", 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175756+0000", 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.33e-06 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175779+0000", 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2578e-05 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175791+0000", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1894e-05 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.175798+0000", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.905e-06 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.179032+0000", 2024-08-17T17:21:58.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032339719999999999 2024-08-17T17:21:58.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.179484+0000", 2024-08-17T17:21:58.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045247100000000003 2024-08-17T17:21:58.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.179626+0000", 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141605 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.184581+0000", 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004954923 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.258378+0000", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073797454999999998 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.258547+0000", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016907399999999999 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.258600+0000", 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2905999999999998e-05 2024-08-17T17:21:58.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.258641+0000", 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0985999999999997e-05 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:124922 5.8 5:1cdcdd28:::benchmark_data_smithi064_147470_object124921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:54.431603+0000", 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.0850127040000004, 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082907817999999994, 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:21:58.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124922 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431603+0000", 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431603+0000", 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431604+0000", 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6449999999999999e-06 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:21:58.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431627+0000", 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3153e-05 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431710+0000", 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2225999999999995e-05 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.431720+0000", 2024-08-17T17:21:58.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0124e-05 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.435816+0000", 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040959270000000001 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.435952+0000", 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013634500000000001 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.436081+0000", 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012869100000000001 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.474280+0000", 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038198850999999999 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:58.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.474325+0000", 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5323e-05 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.514043+0000", 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039717897000000002 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.514118+0000", 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5073000000000004e-05 2024-08-17T17:21:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.514510+0000", 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039256300000000002 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:21:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:21:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:58.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:58.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:58.827 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:58.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 128943 128927 94.7806 56.1875 0.0155604 0.0105459 2024-08-17T17:21:58.929 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:58.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.015 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:21:59.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:59.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:59.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:59.209 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:59.243 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:59.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:59.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:21:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:21:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:21:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:59.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:59.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:59.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:59.659 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:21:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:59.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:59.800 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:59.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 129910 129894 94.3813 60.4375 0.0180958 0.0105905 2024-08-17T17:21:59.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:21:59.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:00.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:00.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:00.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:00.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:00.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:00.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:00.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:00.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:00 smithi175 ceph-mon[101434]: pgmap v349: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:22:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:00 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:00 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:22:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:00 smithi175 ceph-mon[101434]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:22:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[104516]: pgmap v349: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:22:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[104516]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[109246]: pgmap v349: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4090085564' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:00.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:00 smithi064 ceph-mon[109246]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:22:00.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.497 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:00.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:00.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:22:00.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.810 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:00.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:00.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 130853 130837 93.9739 58.9375 0.00991575 0.0106358 2024-08-17T17:22:00.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.940 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:00.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:01.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:01.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:01.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:01.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:01.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:01.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:01.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:01.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:01.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:01 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:22:01.389+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e120 definitely_dead 0 2024-08-17T17:22:01.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:01.790 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:01.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 131798 131782 93.5772 59.0625 0.0227682 0.0106817 2024-08-17T17:22:01.912 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:01.930 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:01.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:02.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:02.033 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.4. 2024-08-17T17:22:02.075 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[109246]: pgmap v351: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[109246]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[104516]: pgmap v351: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:22:02.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:02 smithi064 ceph-mon[104516]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:22:02.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:02.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:02.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:02.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:02 smithi175 ceph-mon[101434]: pgmap v351: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-17T17:22:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:22:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:02 smithi175 ceph-mon[101434]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:22:02.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:02.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:02.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:02.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:02.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:02.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:02.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:02.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:02.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:02.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 131941 131925 92.6263 8.9375 0.0049287 0.0106906 2024-08-17T17:22:02.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:02.948 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:03.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:03.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.258 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:22:03.258 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:03 smithi064 ceph-mon[104516]: osdmap e121: 8 total, 8 up, 7 in 2024-08-17T17:22:03.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:22:03.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:03 smithi064 ceph-mon[109246]: osdmap e121: 8 total, 8 up, 7 in 2024-08-17T17:22:03.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:03.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:03.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/994808390' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:22:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:03 smithi175 ceph-mon[101434]: osdmap e121: 8 total, 8 up, 7 in 2024-08-17T17:22:03.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:03.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:03.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:03.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:03.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:03.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:03.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:03.824 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:03.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:22:03.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 90 16 131941 131925 91.5972 0 - 0.0106906 2024-08-17T17:22:03.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:04.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[104516]: pgmap v353: 89 pgs: 6 peering, 83 active+clean; 8.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:22:04.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:22:04.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[104516]: osdmap e122: 8 total, 8 up, 7 in 2024-08-17T17:22:04.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:03] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:22:04.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[109246]: pgmap v353: 89 pgs: 6 peering, 83 active+clean; 8.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:22:04.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:22:04.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:04 smithi064 ceph-mon[109246]: osdmap e122: 8 total, 8 up, 7 in 2024-08-17T17:22:04.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:04.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:04.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:04.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:04.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:04.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:04 smithi175 ceph-mon[101434]: pgmap v353: 89 pgs: 6 peering, 83 active+clean; 8.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:22:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:04 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:22:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:04 smithi175 ceph-mon[101434]: osdmap e122: 8 total, 8 up, 7 in 2024-08-17T17:22:04.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:04.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:04.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:04.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:04.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:04.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:04.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.6551 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 131941 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 90.9636 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 42.7015 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 236.312 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1455 2024-08-17T17:22:04.896 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 683.225 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3781 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0109696 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0265202 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.49236 2024-08-17T17:22:04.897 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00111021 2024-08-17T17:22:04.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:22:04.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:05.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:05.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:05.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:05.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[104516]: osdmap e123: 8 total, 8 up, 7 in 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[104516]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 120'8248 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[104516]: 5.c starting backfill to osd.6 from (0'0,0'0] MAX to 120'8327 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[104516]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 120'8391 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[109246]: osdmap e123: 8 total, 8 up, 7 in 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[109246]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 120'8248 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[109246]: 5.c starting backfill to osd.6 from (0'0,0'0] MAX to 120'8327 2024-08-17T17:22:05.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:05 smithi064 ceph-mon[109246]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 120'8391 2024-08-17T17:22:05.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:05 smithi175 ceph-mon[101434]: osdmap e123: 8 total, 8 up, 7 in 2024-08-17T17:22:05.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:05 smithi175 ceph-mon[101434]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 120'8248 2024-08-17T17:22:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:05 smithi175 ceph-mon[101434]: 5.c starting backfill to osd.6 from (0'0,0'0] MAX to 120'8327 2024-08-17T17:22:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:05 smithi175 ceph-mon[101434]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 120'8391 2024-08-17T17:22:05.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:05.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:05.650 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:05.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:05.734 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:05.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:05.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:05.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:05.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:06.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:06.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132619 5.0 e124/122)", 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.105560+0000", 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026312249999999999, 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026342936000000001, 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105560+0000", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105560+0000", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105560+0000", 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5900000000000001e-07 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105564+0000", 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7529999999999999e-06 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105571+0000", 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.267e-06 2024-08-17T17:22:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105575+0000", 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8759999999999998e-06 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132624 5.0 e125/122)", 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.119638+0000", 2024-08-17T17:22:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012234583, 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012307818, 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119638+0000", 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119638+0000", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119640+0000", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8369999999999999e-06 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119650+0000", 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0550000000000001e-05 2024-08-17T17:22:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119674+0000", 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3518000000000001e-05 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119686+0000", 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2558000000000001e-05 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132601 5.0 5:05f3b686:::benchmark_data_smithi064_147470_object657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.112176178, 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11229767, 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132601 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019698+0000", 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8640000000000001e-06 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019705+0000", 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.55e-06 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019726+0000", 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0879000000000001e-05 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019736+0000", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0013000000000001e-05 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029209+0000", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094732259999999995 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029369+0000", 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159946 2024-08-17T17:22:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029489+0000", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001198 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.038601+0000", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091122100000000008 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.048008+0000", 2024-08-17T17:22:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094067490000000007 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.120106+0000", 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.072098185999999995 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.126154+0000", 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060479540000000004 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132641 5.0 5.b77b0410 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e125)", 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0060866160000000004, 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062921069999999999, 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132641 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125788+0000", 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7320000000000001e-06 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125795+0000", 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9070000000000001e-06 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125820+0000", 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5303000000000002e-05 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125830+0000", 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0322999999999999e-05 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132603 5.0 5:09704f36:::benchmark_data_smithi064_147470_object659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.112119863, 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.112377403, 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132603 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6100000000000001e-07 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019756+0000", 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8030000000000001e-06 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019764+0000", 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7880000000000007e-06 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019770+0000", 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7189999999999998e-06 2024-08-17T17:22:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.033646+0000", 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013876555 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.033816+0000", 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000170316 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.034257+0000", 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044021300000000001 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.048169+0000", 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013912598999999999 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050618+0000", 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024488000000000001 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132647 5.1 5:8448e17e:::benchmark_data_smithi064_147470_object703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.131414+0000", 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00045878800000000001, 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00078852499999999999, 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132647 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131414+0000", 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131414+0000", 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131416+0000", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0760000000000001e-06 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131423+0000", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3529999999999997e-06 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131446+0000", 2024-08-17T17:22:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2660999999999999e-05 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131456+0000", 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013e-05 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131830+0000", 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000373984 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132604 5.0 5:0d481b14:::benchmark_data_smithi064_147470_object660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.11209469, 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.112481489, 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132604 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6100000000000001e-07 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019781+0000", 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.176e-06 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019787+0000", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5230000000000001e-06 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019792+0000", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.215e-06 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.038806+0000", 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019013690999999999 2024-08-17T17:22:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.039272+0000", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00046591100000000001 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.039719+0000", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044674999999999999 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050594+0000", 2024-08-17T17:22:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010875793999999999 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050629+0000", 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.485e-05 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:132645 5.a e125/122 5:5ef75c4f:::benchmark_data_smithi064_147470_object701:head v 125'8281, mlcod=0'0)", 2024-08-17T17:22:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.131688+0000", 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00018485999999999999, 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065010299999999997, 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131688+0000", 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131688+0000", 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131689+0000", 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.23e-07 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131696+0000", 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2119999999999997e-06 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131741+0000", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5339000000000001e-05 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131749+0000", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5499999999999997e-06 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131839+0000", 2024-08-17T17:22:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9899000000000001e-05 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.131872+0000", 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3797000000000002e-05 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132609 5.0 5:01acadfa:::benchmark_data_smithi064_147470_object665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.107972981, 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.108486966, 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132609 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3799999999999996e-07 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023904+0000", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3759999999999999e-06 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023912+0000", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7560000000000005e-06 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023919+0000", 2024-08-17T17:22:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3779999999999998e-06 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044123+0000", 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020204610000000001 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044276+0000", 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015251199999999999 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044428+0000", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015169399999999999 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050644+0000", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062167129999999996 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052141+0000", 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014969130000000001 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132619 5.0 5:05c1ac25:::benchmark_data_smithi064_147470_object675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.079505712000000006, 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.080099001000000003, 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132619 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052368+0000", 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-06 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052374+0000", 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4110000000000004e-06 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052384+0000", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9790000000000005e-06 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052390+0000", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3759999999999996e-06 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052444+0000", 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4432999999999999e-05 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052599+0000", 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015491999999999999 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052737+0000", 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137916 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.057166+0000", 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044290689999999999 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.066402+0000", 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092355049999999998 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132624 5.0 5:0b125aaf:::benchmark_data_smithi064_147470_object680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.061345069000000002, 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062021837000000003, 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132624 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070529+0000", 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7540000000000001e-06 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070534+0000", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070548+0000", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3749e-05 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070555+0000", 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.199e-06 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076401+0000", 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058451279999999998 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076591+0000", 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019039799999999999 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076781+0000", 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018987499999999999 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.083761+0000", 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006980193 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.085668+0000", 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001906818 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132601 5.0 e124/122 ondisk, result = 0)", 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.105254+0000", 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026618971000000002, 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027370457000000001, 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105254+0000", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105254+0000", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105254+0000", 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0699999999999999e-07 2024-08-17T17:22:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105256+0000", 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9850000000000001e-06 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105264+0000", 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9449999999999993e-06 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105268+0000", 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.799e-06 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.120058+0000", 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014790477999999999 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.120083+0000", 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4882e-05 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132603 5.0 e124/122)", 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.105273+0000", 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026599462000000001, 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027398987999999999, 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105273+0000", 2024-08-17T17:22:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105273+0000", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105273+0000", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6899999999999999e-07 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105275+0000", 2024-08-17T17:22:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6789999999999999e-06 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105282+0000", 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9179999999999997e-06 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105285+0000", 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5209999999999998e-06 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132604 5.0 e124/122)", 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.105515+0000", 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026357255999999999, 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027191745, 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105515+0000", 2024-08-17T17:22:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105515+0000", 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105516+0000", 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1200000000000003e-07 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105519+0000", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4580000000000002e-06 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105531+0000", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1421e-05 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105536+0000", 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5269999999999998e-06 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:132609 5.0 e124/122)", 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.105545+0000", 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026327099999999999, 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027206373999999998, 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105545+0000", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105545+0000", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105546+0000", 2024-08-17T17:22:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8099999999999999e-07 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105548+0000", 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9139999999999998e-06 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105552+0000", 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5759999999999999e-06 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.105556+0000", 2024-08-17T17:22:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5710000000000002e-06 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 15 2024-08-17T17:22:06.202 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:22:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.203 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:06.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:06 smithi064 ceph-mon[104516]: pgmap v356: 89 pgs: 4 remapped+peering, 6 peering, 79 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:22:06.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:06 smithi064 ceph-mon[104516]: osdmap e124: 8 total, 8 up, 7 in 2024-08-17T17:22:06.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:06 smithi064 ceph-mon[109246]: pgmap v356: 89 pgs: 4 remapped+peering, 6 peering, 79 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:22:06.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:06 smithi064 ceph-mon[109246]: osdmap e124: 8 total, 8 up, 7 in 2024-08-17T17:22:06.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:06.329 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:06.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:06 smithi175 ceph-mon[101434]: pgmap v356: 89 pgs: 4 remapped+peering, 6 peering, 79 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:22:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:06 smithi175 ceph-mon[101434]: osdmap e124: 8 total, 8 up, 7 in 2024-08-17T17:22:06.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:06.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:06.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:06.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:06.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:06.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:06.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:06.810 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:06.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:06.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:06.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:06.892 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:06.892 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:06.929 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:06.929 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:06.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.962 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:06.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:06.989 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:06.989 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:07.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:07.039 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:07.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:07.075 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:07.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:07.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:07 smithi064 ceph-mon[104516]: osdmap e125: 8 total, 8 up, 7 in 2024-08-17T17:22:07.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2204585903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:07 smithi064 ceph-mon[109246]: osdmap e125: 8 total, 8 up, 7 in 2024-08-17T17:22:07.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2204585903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:07.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:07.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:07.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:07 smithi175 ceph-mon[101434]: osdmap e125: 8 total, 8 up, 7 in 2024-08-17T17:22:07.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2204585903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:07.497 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:07.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:07.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:07.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:07.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:07.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:08.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:22:08.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:08.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:08.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:08.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:08.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:08.359 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:08 smithi175 ceph-mon[101434]: pgmap v359: 89 pgs: 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 7 peering, 74 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 255 B/s rd, 19 MiB/s wr, 403 op/s; 24777/394680 objects degraded (6.278%); 8326/394680 objects misplaced (2.110%); 1.6 MiB/s, 2 keys/s, 15 objects/s recovering 2024-08-17T17:22:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:08 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 24777/394680 objects degraded (6.278%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:08 smithi064 ceph-mon[104516]: pgmap v359: 89 pgs: 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 7 peering, 74 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 255 B/s rd, 19 MiB/s wr, 403 op/s; 24777/394680 objects degraded (6.278%); 8326/394680 objects misplaced (2.110%); 1.6 MiB/s, 2 keys/s, 15 objects/s recovering 2024-08-17T17:22:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:08 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 24777/394680 objects degraded (6.278%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:08 smithi064 ceph-mon[109246]: pgmap v359: 89 pgs: 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 7 peering, 74 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 255 B/s rd, 19 MiB/s wr, 403 op/s; 24777/394680 objects degraded (6.278%); 8326/394680 objects misplaced (2.110%); 1.6 MiB/s, 2 keys/s, 15 objects/s recovering 2024-08-17T17:22:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:08 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 24777/394680 objects degraded (6.278%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:08.461 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:08.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:08.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:08.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:08.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:08.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:08.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:08.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:08.942 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:22:08.942 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:22:08.943 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-08-17T17:22:08.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd in 4 2024-08-17T17:22:08.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:09.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:09.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:09 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:22:09.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:09.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:09 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:22:09.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:09 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:22:09.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:09.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:09.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:09.810 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:09.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:09.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:10.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.078 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:10.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:10.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:10.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:10.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:10.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:10 smithi175 ceph-mon[101434]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 80 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 179 B/s rd, 13 MiB/s wr, 384 op/s; 33102/392955 objects degraded (8.424%); 32611/392955 objects misplaced (8.299%); 2.7 MiB/s, 1 keys/s, 38 objects/s recovering 2024-08-17T17:22:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[104516]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 80 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 179 B/s rd, 13 MiB/s wr, 384 op/s; 33102/392955 objects degraded (8.424%); 32611/392955 objects misplaced (8.299%); 2.7 MiB/s, 1 keys/s, 38 objects/s recovering 2024-08-17T17:22:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[109246]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 80 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 179 B/s rd, 13 MiB/s wr, 384 op/s; 33102/392955 objects degraded (8.424%); 32611/392955 objects misplaced (8.299%); 2.7 MiB/s, 1 keys/s, 38 objects/s recovering 2024-08-17T17:22:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:10.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:10.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:10.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:10.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:10.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:10.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:10.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:10.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:11.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:11.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:11.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:11.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:11.254 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:11.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:11.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:11.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:11.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:11.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:11.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:11.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:11.702 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:11.715 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:11.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:11.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:11.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:11.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:12.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:12.035 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:12.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:12.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:12.281 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:12.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:12.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:12.356 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:12 smithi064 ceph-mon[104516]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 6.9 MiB/s wr, 341 op/s; 33102/391272 objects degraded (8.460%); 32611/391272 objects misplaced (8.335%); 2.3 MiB/s, 1 keys/s, 32 objects/s recovering 2024-08-17T17:22:12.357 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:12 smithi064 ceph-mon[109246]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 6.9 MiB/s wr, 341 op/s; 33102/391272 objects degraded (8.460%); 32611/391272 objects misplaced (8.335%); 2.3 MiB/s, 1 keys/s, 32 objects/s recovering 2024-08-17T17:22:12.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:12.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:12 smithi175 ceph-mon[101434]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 8.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 6.9 MiB/s wr, 341 op/s; 33102/391272 objects degraded (8.460%); 32611/391272 objects misplaced (8.335%); 2.3 MiB/s, 1 keys/s, 32 objects/s recovering 2024-08-17T17:22:12.430 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:12.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:12.555 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:12.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:12.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:12.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:12.936 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:13.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:13.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:13.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:13.164 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:22:13.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:13.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:13.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:13.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/916509795' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:13 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33102/391272 objects degraded (8.460%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:13.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/916509795' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33102/391272 objects degraded (8.460%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/916509795' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33102/391272 objects degraded (8.460%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:13.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:13.548 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:13.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:13.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:13.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:13.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:13.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:13.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:14.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:14.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:14.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[104516]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 511 op/s; 33195/385845 objects degraded (8.603%); 32283/385845 objects misplaced (8.367%); 2.5 MiB/s, 1 keys/s, 36 objects/s recovering 2024-08-17T17:22:14.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:14.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:14.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:13] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-08-17T17:22:14.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:14.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:14.326 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:14.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:14 smithi175 ceph-mon[101434]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 511 op/s; 33195/385845 objects degraded (8.603%); 32283/385845 objects misplaced (8.367%); 2.5 MiB/s, 1 keys/s, 36 objects/s recovering 2024-08-17T17:22:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:14 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:14 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:14 smithi175 ceph-mon[101434]: osdmap e126: 8 total, 8 up, 7 in 2024-08-17T17:22:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[104516]: osdmap e126: 8 total, 8 up, 7 in 2024-08-17T17:22:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[109246]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 511 op/s; 33195/385845 objects degraded (8.603%); 32283/385845 objects misplaced (8.367%); 2.5 MiB/s, 1 keys/s, 36 objects/s recovering 2024-08-17T17:22:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:14 smithi064 ceph-mon[109246]: osdmap e126: 8 total, 8 up, 7 in 2024-08-17T17:22:14.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:14.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:14.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:14.712 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:14.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:14.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:14.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:14.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:15.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:15.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:15.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:15.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:15.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:15.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:15.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:15.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:15.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:15.619 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:15.619 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:15.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:15.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:15.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:15.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:15.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:16.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:16.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:16.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:16.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:16.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:16.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:16.425 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:16 smithi064 ceph-mon[104516]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 118 B/s rd, 1.8 MiB/s wr, 460 op/s; 33195/384153 objects degraded (8.641%); 31897/384153 objects misplaced (8.303%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:22:16.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:16 smithi064 ceph-mon[109246]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 118 B/s rd, 1.8 MiB/s wr, 460 op/s; 33195/384153 objects degraded (8.641%); 31897/384153 objects misplaced (8.303%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:22:16.527 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:16.551 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:16.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:16 smithi175 ceph-mon[101434]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 118 B/s rd, 1.8 MiB/s wr, 460 op/s; 33195/384153 objects degraded (8.641%); 31897/384153 objects misplaced (8.303%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:22:16.680 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:16.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:16.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:16.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:16.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:16.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:17.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:17.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:17.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:17.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:17.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:17.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:17.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:17.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:17.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:17.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:17.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:17.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:17.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:17.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:18.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:18.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.059 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:18.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:18.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:18.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:18 smithi064 ceph-mon[104516]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 KiB/s wr, 546 op/s; 33195/378291 objects degraded (8.775%); 31712/378291 objects misplaced (8.383%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:22:18.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33195/378291 objects degraded (8.775%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:18.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:18 smithi064 ceph-mon[109246]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 KiB/s wr, 546 op/s; 33195/378291 objects degraded (8.775%); 31712/378291 objects misplaced (8.383%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:22:18.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33195/378291 objects degraded (8.775%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:18.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:18.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:18.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:18.512 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:18.512 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:18.545 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:18.545 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:18.567 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:18.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:18.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:18.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:18.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:18.613 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:18.613 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:18.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:18.646 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:18.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:18 smithi175 ceph-mon[101434]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 19 KiB/s wr, 546 op/s; 33195/378291 objects degraded (8.775%); 31712/378291 objects misplaced (8.383%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:22:18.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33195/378291 objects degraded (8.775%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:18.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:18.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:18.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.712 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:18.712 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:18.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.748 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:18.748 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:18.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:18.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:18.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:18.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:18.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:19.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:19.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:19.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:19.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:19.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:19.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:19.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:22:19.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/977735041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/977735041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:19 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:22:19.337+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e126 definitely_dead 0 2024-08-17T17:22:19.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:19.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:19.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:19.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/977735041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:19.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:19.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:19.945 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:19.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:20.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:20.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:20.204 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.4. 2024-08-17T17:22:20.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.253 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:20.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:20.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:20.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:22:20.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:20.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[104516]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33316/375513 objects degraded (8.872%); 31160/375513 objects misplaced (8.298%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:22:20.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4285304844' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[109246]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33316/375513 objects degraded (8.872%); 31160/375513 objects misplaced (8.298%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:22:20.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4285304844' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:20 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:20.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:20.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:20.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:20.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:20 smithi175 ceph-mon[101434]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 7.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33316/375513 objects degraded (8.872%); 31160/375513 objects misplaced (8.298%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:22:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:20 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4285304844' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:20 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T17:22:20.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:20.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:20.778 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:20.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:20.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:20.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.998 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:20.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:21.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:21.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:21.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:21.254 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-08-17T17:22:21.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:21.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:21.396 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:21.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:21.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:21.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:21.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:21.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T17:22:21.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:21 smithi175 ceph-mon[101434]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:22:21.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:21 smithi175 ceph-mon[101434]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:22:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T17:22:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[104516]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:22:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[104516]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:22:21.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T17:22:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[109246]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:22:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:21 smithi064 ceph-mon[109246]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:22:21.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:21.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:21.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:21.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:21.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:22.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:22.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:22.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:22.136 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:22.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:22.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:22.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:22.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[104516]: pgmap v368: 89 pgs: 1 remapped+peering, 21 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 60 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 25112/374127 objects degraded (6.712%); 31160/374127 objects misplaced (8.329%); 567 KiB/s, 8 objects/s recovering 2024-08-17T17:22:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T17:22:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[104516]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:22:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[109246]: pgmap v368: 89 pgs: 1 remapped+peering, 21 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 60 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 25112/374127 objects degraded (6.712%); 31160/374127 objects misplaced (8.329%); 567 KiB/s, 8 objects/s recovering 2024-08-17T17:22:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T17:22:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:22 smithi064 ceph-mon[109246]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:22:22.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:22.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:22.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:22 smithi175 ceph-mon[101434]: pgmap v368: 89 pgs: 1 remapped+peering, 21 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 60 active+clean; 7.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 25112/374127 objects degraded (6.712%); 31160/374127 objects misplaced (8.329%); 567 KiB/s, 8 objects/s recovering 2024-08-17T17:22:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:22 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T17:22:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:22 smithi175 ceph-mon[101434]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:22:22.672 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:22.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:22.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:22.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:22.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:23.085 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:23.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:23.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:23.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:23.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:23.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:23.490 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:23.591 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:23.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25112/374127 objects degraded (6.712%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25112/374127 objects degraded (6.712%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:23.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25112/374127 objects degraded (6.712%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:23.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:23.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:23.865 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:23.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:24.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:24.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:24.091 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:23] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-08-17T17:22:24.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:24.166 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:24.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:24.256 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:24.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:24.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:24 smithi064 ceph-mon[104516]: pgmap v371: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 9409/369699 objects degraded (2.545%); 15489/369699 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:22:24.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:24 smithi064 ceph-mon[109246]: pgmap v371: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 9409/369699 objects degraded (2.545%); 15489/369699 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:22:24.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:24.496 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:24.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:24.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:24 smithi175 ceph-mon[101434]: pgmap v371: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 9409/369699 objects degraded (2.545%); 15489/369699 objects misplaced (4.190%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:22:24.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:24.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:24.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:24.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:24.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:24.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:25.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:25.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:25.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:25.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:25.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:25.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:25.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:25.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:25.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:25.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:25.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:25.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:25.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:25.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:25.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:25.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:25.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:25.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:26.026 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[104516]: pgmap v372: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1704/368844 objects degraded (0.462%); 7665/368844 objects misplaced (2.078%); 372 KiB/s, 5 objects/s recovering 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[109246]: pgmap v372: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1704/368844 objects degraded (0.462%); 7665/368844 objects misplaced (2.078%); 372 KiB/s, 5 objects/s recovering 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:26.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:95998 5.0 5:0e7f84b1:::benchmark_data_smithi064_147470_object95997:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.730124+0000", 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 59.479897461999997, 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094063040000000001, 2024-08-17T17:22:26.214 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95998 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730124+0000", 2024-08-17T17:22:26.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730124+0000", 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730126+0000", 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7379999999999999e-06 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730150+0000", 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4423e-05 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730163+0000", 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3237999999999999e-05 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730172+0000", 2024-08-17T17:22:26.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1699999999999997e-06 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.730711+0000", 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053925200000000005 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.731054+0000", 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000343056 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.731484+0000", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000430056 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735453+0000", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039691329999999997 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735516+0000", 2024-08-17T17:22:26.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3305000000000005e-05 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824068+0000", 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.088551699999999997 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824135+0000", 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6950000000000001e-05 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.824187+0000", 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2018999999999998e-05 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131941 5.0 5:0bbb7707:::benchmark_data_smithi064_147470_object131940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.126267053999999, 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.435185814, 2024-08-17T17:22:26.221 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131941 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083756+0000", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9989999999999998e-06 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083779+0000", 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2419999999999999e-05 2024-08-17T17:22:26.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083798+0000", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8740999999999999e-05 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083805+0000", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6480000000000003e-06 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083928+0000", 2024-08-17T17:22:26.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012243700000000001 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083942+0000", 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3985000000000001e-05 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.047013+0000", 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96307144200000006 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.047435+0000", 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042231199999999999 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.498588+0000", 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4511525169999999 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.498745+0000", 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015708900000000001 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.499521+0000", 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00077622900000000002 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.513675+0000", 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014153361999999999 2024-08-17T17:22:26.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.513798+0000", 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012309899999999999 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515148+0000", 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013501419999999999 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515229+0000", 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0810999999999996e-05 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.518940+0000", 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037115809999999998 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132582 5.0 5:08eb4a0b:::benchmark_data_smithi064_147470_object638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:05.985392+0000", 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.224629757999999, 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090946227000000004, 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.230 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132582 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985392+0000", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985392+0000", 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985394+0000", 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1059999999999998e-06 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985401+0000", 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2300000000000002e-06 2024-08-17T17:22:26.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985428+0000", 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6480000000000001e-05 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985439+0000", 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1863e-05 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.985667+0000", 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000227094 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.986132+0000", 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00046511799999999999 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.986700+0000", 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056858500000000003 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:05.998409+0000", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011708532000000001 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.002267+0000", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003857784 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070311+0000", 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068044119 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070392+0000", 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1573e-05 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076338+0000", 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059457429999999999 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132594 5.0 5:041e9e63:::benchmark_data_smithi064_147470_object650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.003194+0000", 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.206827072999999, 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11672486899999999, 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132594 2024-08-17T17:22:26.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003194+0000", 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003194+0000", 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003196+0000", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.782e-06 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003203+0000", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7449999999999998e-06 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003220+0000", 2024-08-17T17:22:26.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6965000000000001e-05 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003228+0000", 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1189999999999997e-06 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003282+0000", 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3977999999999997e-05 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003457+0000", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017531900000000001 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.003914+0000", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045689899999999998 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.008190+0000", 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004275259 2024-08-17T17:22:26.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.009009+0000", 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081940699999999997 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.110048+0000", 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101038752 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.110106+0000", 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.842e-05 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.119919+0000", 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098132240000000006 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132601 5.0 5:05f3b686:::benchmark_data_smithi064_147470_object657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.190325145999999, 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11757521899999999, 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.244 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132601 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019696+0000", 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019698+0000", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8640000000000001e-06 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019705+0000", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.55e-06 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019726+0000", 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0879000000000001e-05 2024-08-17T17:22:26.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019736+0000", 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0013000000000001e-05 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029209+0000", 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094732259999999995 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029369+0000", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159946 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.029489+0000", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001198 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.038601+0000", 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091122100000000008 2024-08-17T17:22:26.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.048008+0000", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094067490000000007 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.120106+0000", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.072098185999999995 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.126154+0000", 2024-08-17T17:22:26.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060479540000000004 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.137272+0000", 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011117841999999999 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132603 5.0 5:09704f36:::benchmark_data_smithi064_147470_object659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.190268831000001, 2024-08-17T17:22:26.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.122725102, 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132603 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:26.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019753+0000", 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6100000000000001e-07 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019756+0000", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8030000000000001e-06 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019764+0000", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7880000000000007e-06 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019770+0000", 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7189999999999998e-06 2024-08-17T17:22:26.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.033646+0000", 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013876555 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.033816+0000", 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000170316 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.034257+0000", 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044021300000000001 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.048169+0000", 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013912598999999999 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050618+0000", 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024488000000000001 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.137333+0000", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.086715423999999999 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.137392+0000", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8786000000000003e-05 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.142478+0000", 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050855379999999997 2024-08-17T17:22:26.256 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132604 5.0 5:0d481b14:::benchmark_data_smithi064_147470_object660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.190243658, 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12769596499999999, 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.257 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132604 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:26.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.259 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:22:26.259 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:22:26.259 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2024-08-17T17:22:26.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd out 5 2024-08-17T17:22:26.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019778+0000", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6100000000000001e-07 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019781+0000", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.176e-06 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019787+0000", 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5230000000000001e-06 2024-08-17T17:22:26.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.019792+0000", 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.215e-06 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.038806+0000", 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019013690999999999 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.039272+0000", 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00046591100000000001 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.039719+0000", 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044674999999999999 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050594+0000", 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010875793999999999 2024-08-17T17:22:26.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050629+0000", 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.485e-05 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.142544+0000", 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.091914923999999995 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.142600+0000", 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5912000000000003e-05 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.147474+0000", 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048737579999999997 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132609 5.0 5:01acadfa:::benchmark_data_smithi064_147470_object665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.265 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.186121949, 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13051273199999999, 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132609 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023900+0000", 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3799999999999996e-07 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023904+0000", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3759999999999999e-06 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023912+0000", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7560000000000005e-06 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.023919+0000", 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3779999999999998e-06 2024-08-17T17:22:26.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044123+0000", 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020204610000000001 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044276+0000", 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015251199999999999 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.044428+0000", 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015169399999999999 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.050644+0000", 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062167129999999996 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052141+0000", 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014969130000000001 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.147602+0000", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.095460352999999998 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.147657+0000", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4935999999999997e-05 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.154412+0000", 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067557529999999998 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.271 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132619 5.0 5:05c1ac25:::benchmark_data_smithi064_147470_object675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.15765468, 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10710808400000001, 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132619 2024-08-17T17:22:26.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052367+0000", 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052368+0000", 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-06 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052374+0000", 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4110000000000004e-06 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052384+0000", 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9790000000000005e-06 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052390+0000", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3759999999999996e-06 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052444+0000", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4432999999999999e-05 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052599+0000", 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015491999999999999 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.052737+0000", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137916 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.057166+0000", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044290689999999999 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.066402+0000", 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092355049999999998 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.154533+0000", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.088131686000000001 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.154592+0000", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8347000000000002e-05 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.159475+0000", 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048831320000000001 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.277 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132624 5.0 5:0b125aaf:::benchmark_data_smithi064_147470_object680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.139494036999999, 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.099186498999999997, 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.278 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132624 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070527+0000", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070529+0000", 2024-08-17T17:22:26.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7540000000000001e-06 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070534+0000", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070548+0000", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3749e-05 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.070555+0000", 2024-08-17T17:22:26.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.199e-06 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076401+0000", 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058451279999999998 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076591+0000", 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019039799999999999 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.076781+0000", 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018987499999999999 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.083761+0000", 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006980193 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.085668+0000", 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001906818 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.159548+0000", 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073879951999999999 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.159610+0000", 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2401000000000001e-05 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.169714+0000", 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010103802 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:132641 5.0 5:0820deed:::benchmark_data_smithi064_147470_object697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:26.284 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.084235584000002, 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.088007884999999994, 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132641 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:26.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125786+0000", 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125788+0000", 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7320000000000001e-06 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125795+0000", 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9070000000000001e-06 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125820+0000", 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5303000000000002e-05 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.125830+0000", 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0322999999999999e-05 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.171176+0000", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045346166 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.171342+0000", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016560500000000001 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.171539+0000", 2024-08-17T17:22:26.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000196696 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.180546+0000", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090074839999999996 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.186564+0000", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060181970000000003 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.201772+0000", 2024-08-17T17:22:26.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015207764 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.201834+0000", 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2019000000000004e-05 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:06.213794+0000", 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011959689000000001 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.290 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140126 5.0 5:074f9d75:::benchmark_data_smithi064_147470_object8182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:19.008953+0000", 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.2010685729999997, 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090161577000000007, 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140126 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.291 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008953+0000", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008953+0000", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008955+0000", 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0559999999999999e-06 2024-08-17T17:22:26.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008962+0000", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.075e-06 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008985+0000", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2832000000000001e-05 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.008994+0000", 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9970000000000004e-06 2024-08-17T17:22:26.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.022852+0000", 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013858583000000001 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.023017+0000", 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164866 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.023172+0000", 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015498000000000001 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.031112+0000", 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079391649999999998 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.031331+0000", 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000219874 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.093997+0000", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062666074000000002 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.094077+0000", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9820999999999994e-05 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.099115+0000", 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050372539999999997 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140130 5.0 5:0239b01b:::benchmark_data_smithi064_147470_object8186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:19.013144+0000", 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.1968775369999998, 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093150866999999998, 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140130 2024-08-17T17:22:26.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013144+0000", 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013144+0000", 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013146+0000", 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8509999999999999e-06 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013151+0000", 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3460000000000001e-06 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013167+0000", 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5838e-05 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.013173+0000", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4740000000000002e-06 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.027066+0000", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013892557999999999 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.027237+0000", 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017114100000000001 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.027421+0000", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018413300000000001 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.038416+0000", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010994359 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.039141+0000", 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072488700000000001 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.099311+0000", 2024-08-17T17:22:26.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060170045999999998 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.099657+0000", 2024-08-17T17:22:26.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034637900000000001 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.106295+0000", 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066378549999999998 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140655 5.0 5:0b0c9538:::benchmark_data_smithi064_147470_object8711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:19.966863+0000", 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.2431582929999996, 2024-08-17T17:22:26.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090372983000000004, 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140655 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966863+0000", 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966863+0000", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966865+0000", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5880000000000001e-06 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966871+0000", 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9920000000000004e-06 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966890+0000", 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9040999999999999e-05 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.966899+0000", 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3190000000000001e-06 2024-08-17T17:22:26.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.972103+0000", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052037020000000002 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.972574+0000", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047091899999999998 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.973079+0000", 2024-08-17T17:22:26.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050498100000000005 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.987482+0000", 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014402984000000001 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.987578+0000", 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6237999999999996e-05 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.047682+0000", 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060104156999999998 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.047740+0000", 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7552000000000001e-05 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.057236+0000", 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094965099999999997 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140658 5.0 5:0acc5644:::benchmark_data_smithi064_147470_object8714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:19.967606+0000", 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.242416017, 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103623574, 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140658 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.311 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967606+0000", 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967606+0000", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967607+0000", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.204e-06 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967612+0000", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3800000000000002e-06 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967629+0000", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7246000000000001e-05 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.967638+0000", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5329999999999994e-06 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.982505+0000", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014867389999999999 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.982919+0000", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041396400000000001 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.983372+0000", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000453251 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.990128+0000", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067552070000000001 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.991244+0000", 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001116438 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.065899+0000", 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074654896999999998 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.065961+0000", 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1562000000000005e-05 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.071229+0000", 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052685020000000004 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140798 5.0 5:06e39447:::benchmark_data_smithi064_147470_object8854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.9554921509999996, 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1158812280000001, 2024-08-17T17:22:26.318 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140798 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254531+0000", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.499e-06 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254537+0000", 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5719999999999999e-06 2024-08-17T17:22:26.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254555+0000", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7589999999999999e-05 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254563+0000", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2360000000000004e-06 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254638+0000", 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4779999999999999e-05 2024-08-17T17:22:26.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254651+0000", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2469e-05 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.265920+0000", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.011269057 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.266255+0000", 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033558699999999998 2024-08-17T17:22:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.266541+0000", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028578399999999997 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.274465+0000", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079242489999999995 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370009+0000", 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.095543367000000004 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370205+0000", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195995 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370268+0000", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3637000000000001e-05 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370411+0000", 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014240600000000001 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.324 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140799 5.0 5:03f574a7:::benchmark_data_smithi064_147470_object8855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.9498342080000004, 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.114930542, 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140799 2024-08-17T17:22:26.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260189+0000", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.393e-06 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260193+0000", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.702e-06 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260202+0000", 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4330000000000004e-06 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260209+0000", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3830000000000003e-06 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.273511+0000", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013301453 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.273522+0000", 2024-08-17T17:22:26.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0890000000000001e-05 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271047+0000", 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99752509700000003 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271196+0000", 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014981400000000001 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:22:26.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271334+0000", 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013721299999999999 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370162+0000", 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098828572000000003 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370457+0000", 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029438799999999997 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374859+0000", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044020370000000001 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374920+0000", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.156e-05 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.375118+0000", 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019760700000000001 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.331 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:140798 5.0 e128/127 ondisk, result = 0)", 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.275666+0000", 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.9343558820000002, 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094352864999999994, 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275666+0000", 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275666+0000", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275666+0000", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0399999999999995e-07 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275671+0000", 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6700000000000002e-06 2024-08-17T17:22:26.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275703+0000", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1730000000000003e-05 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275711+0000", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8639999999999996e-06 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.369992+0000", 2024-08-17T17:22:26.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094281698999999997 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370002+0000", 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4520000000000003e-06 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370019+0000", 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6745999999999999e-05 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.335 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.34626.0:140799 5.0 e128/127 ondisk, result = 0)", 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.280062+0000", 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.9299594979999997, 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090107392999999994, 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280062+0000", 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280062+0000", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280063+0000", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5899999999999995e-07 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280068+0000", 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4110000000000004e-06 2024-08-17T17:22:26.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280080+0000", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1569999999999999e-05 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280091+0000", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1447999999999999e-05 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370148+0000", 2024-08-17T17:22:26.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090056382000000004 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370156+0000", 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0209999999999999e-06 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370169+0000", 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3801999999999999e-05 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.339 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140803 5.0 5:0abd7c66:::benchmark_data_smithi064_147470_object8859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.282956+0000", 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.9270652789999998, 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.099524801999999996, 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:26.340 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140803 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282956+0000", 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282956+0000", 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:22:26.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282958+0000", 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.73e-06 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282964+0000", 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3629999999999999e-06 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282978+0000", 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4037e-05 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282988+0000", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8509999999999994e-06 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370492+0000", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.087503697000000005 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370643+0000", 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151297 2024-08-17T17:22:26.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370771+0000", 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127463 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.375662+0000", 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048914140000000002 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377109+0000", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001446524 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.382200+0000", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050908799999999999 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.382278+0000", 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8534000000000004e-05 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:22:26.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.382481+0000", 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000203012 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:22:26.346 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:22:26.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:26 smithi175 ceph-mon[101434]: pgmap v372: 89 pgs: 1 activating+undersized+remapped, 3 remapped+peering, 1 activating, 23 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 57 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 1704/368844 objects degraded (0.462%); 7665/368844 objects misplaced (2.078%); 372 KiB/s, 5 objects/s recovering 2024-08-17T17:22:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:26.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:26.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:26.641 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:26.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:26.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:26.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:26.915 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:27.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:27.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:27.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:27.186 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:27.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:27 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2024-08-17T17:22:27.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:27 smithi064 ceph-mon[104516]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:22:27.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:27 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2024-08-17T17:22:27.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:27 smithi064 ceph-mon[109246]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:22:27.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:27.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:27.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:27 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2024-08-17T17:22:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:27 smithi175 ceph-mon[101434]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:22:27.416 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:27.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:27.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:27.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:27.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:27.685 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:27.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:27.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:27.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:28.058 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:28.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:28.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:28.297 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:28.339 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:28 smithi175 ceph-mon[101434]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2231/360948 objects degraded (0.618%); 371 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:28 smithi175 ceph-mon[101434]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:22:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2231/360948 objects degraded (0.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[104516]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2231/360948 objects degraded (0.618%); 371 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[104516]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:22:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2231/360948 objects degraded (0.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[109246]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2231/360948 objects degraded (0.618%); 371 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[109246]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:22:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2231/360948 objects degraded (0.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:28.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:28.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:28.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:28.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:28.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:28.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:28.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:28.871 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:28.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:29.000 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:29.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:29.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:29.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:29.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:29.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:29.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:29.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:29.272 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:29.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:29.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:29.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:29.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:29.345 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:29.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:29.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:29.379 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:29.380 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:29.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:29.419 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:29.419 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:29.432 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:22:29.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:29.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:29.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:29.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:29.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:29.839 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:29.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:29.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:30.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:30.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:30.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:30.229 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:30.241 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:30 smithi175 ceph-mon[101434]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2231/357906 objects degraded (0.623%); 372 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:30.241 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/841753120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:30.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:30.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:30.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:30.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:30 smithi064 ceph-mon[104516]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2231/357906 objects degraded (0.623%); 372 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/841753120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:30 smithi064 ceph-mon[109246]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2231/357906 objects degraded (0.623%); 372 KiB/s, 1 keys/s, 92 objects/s recovering 2024-08-17T17:22:30.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/841753120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:30.532 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:30.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:30.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:30.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:30.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:30.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:30.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:30.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:30.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:31.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:31.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:31.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:31.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:31.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:31.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:31.515 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:31.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:31.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:31.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:31.885 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:31.885 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:31.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:31.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:32.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:32.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:32.116 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:22:32.140 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:32.242 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:32.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:32.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:32.408 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:32 smithi175 ceph-mon[101434]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 2231/355845 objects degraded (0.627%); 25 KiB/s, 1 keys/s, 71 objects/s recovering 2024-08-17T17:22:32.408 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:32 smithi175 ceph-mon[101434]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:22:32.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3136210443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:32.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[104516]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 2231/355845 objects degraded (0.627%); 25 KiB/s, 1 keys/s, 71 objects/s recovering 2024-08-17T17:22:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[104516]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3136210443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[109246]: pgmap v377: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 2231/355845 objects degraded (0.627%); 25 KiB/s, 1 keys/s, 71 objects/s recovering 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[109246]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3136210443' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:32.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:32.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:22:32.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:32.678 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:32.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:32.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:32.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:32.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:32.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:33.007 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:33.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:33.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:33.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:33.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[104516]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2231/355845 objects degraded (0.627%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:33.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:33.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[109246]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:22:33.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2231/355845 objects degraded (0.627%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:33.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:33.373 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:33 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:33 smithi175 ceph-mon[101434]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:22:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2231/355845 objects degraded (0.627%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:33.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:33.513 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:33.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:33.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.662 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:33.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:33.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:33.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:33.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:33.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:34.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:34.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:34.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:34.134 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.5. 2024-08-17T17:22:34.134 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:33] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:22:34.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:22:33.877+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e133 definitely_dead 0 2024-08-17T17:22:34.166 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:34.188 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:34.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:34.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:34 smithi175 ceph-mon[101434]: pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 1693/348186 objects degraded (0.486%); 11 KiB/s, 95 objects/s recovering 2024-08-17T17:22:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:22:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:34 smithi175 ceph-mon[101434]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:22:34.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:34.435 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[104516]: pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 1693/348186 objects degraded (0.486%); 11 KiB/s, 95 objects/s recovering 2024-08-17T17:22:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:22:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[104516]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:22:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[109246]: pgmap v380: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 1693/348186 objects degraded (0.486%); 11 KiB/s, 95 objects/s recovering 2024-08-17T17:22:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:22:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:34 smithi064 ceph-mon[109246]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:22:34.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:34.559 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:34.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:34.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:34.802 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:34.802 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:34.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:34.909 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:34.930 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:34.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:34.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:22:35.032 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:35.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:35.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:35.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:35.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:35.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:35.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:35.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:22:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:35 smithi175 ceph-mon[101434]: osdmap e134: 8 total, 8 up, 7 in 2024-08-17T17:22:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:35 smithi175 ceph-mon[101434]: osdmap e135: 8 total, 8 up, 7 in 2024-08-17T17:22:35.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:35.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[104516]: osdmap e134: 8 total, 8 up, 7 in 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[104516]: osdmap e135: 8 total, 8 up, 7 in 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1803042337' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[109246]: osdmap e134: 8 total, 8 up, 7 in 2024-08-17T17:22:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:35 smithi064 ceph-mon[109246]: osdmap e135: 8 total, 8 up, 7 in 2024-08-17T17:22:35.529 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:35.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:35.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:35.660 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:35.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:35.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:35.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:35.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:22:35.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:35.958 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:36.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:36.122 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:36.187 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:36.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:36.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:36.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:36 smithi175 ceph-mon[101434]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 1693/346926 objects degraded (0.488%); 11 KiB/s, 91 objects/s recovering 2024-08-17T17:22:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:22:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:36 smithi175 ceph-mon[101434]: osdmap e136: 8 total, 8 up, 7 in 2024-08-17T17:22:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[104516]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 1693/346926 objects degraded (0.488%); 11 KiB/s, 91 objects/s recovering 2024-08-17T17:22:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:22:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[104516]: osdmap e136: 8 total, 8 up, 7 in 2024-08-17T17:22:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[109246]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 1693/346926 objects degraded (0.488%); 11 KiB/s, 91 objects/s recovering 2024-08-17T17:22:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:22:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:36 smithi064 ceph-mon[109246]: osdmap e136: 8 total, 8 up, 7 in 2024-08-17T17:22:36.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:36.483 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:36.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.490 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:36.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:36.504 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:36.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:36.591 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:36.625 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:36.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:36.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:36.663 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:36.663 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:36.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:36.701 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:36.740 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:36.741 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:36.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:36.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:36.775 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:36.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:36.812 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:36.812 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:36.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:36.937 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:36.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:37.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:37.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:37.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:37.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:37.203 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:37.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:37.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[104516]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 133'9338 2024-08-17T17:22:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[104516]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 133'9507 2024-08-17T17:22:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[104516]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 133'9399 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[104516]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 133'9196 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2412145796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[109246]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 133'9338 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[109246]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 133'9507 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[109246]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 133'9399 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[109246]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 133'9196 2024-08-17T17:22:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2412145796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:37.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:37.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:37.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:37.615 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:37 smithi175 ceph-mon[101434]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 133'9338 2024-08-17T17:22:37.615 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:37 smithi175 ceph-mon[101434]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 133'9507 2024-08-17T17:22:37.616 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:37 smithi175 ceph-mon[101434]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 133'9399 2024-08-17T17:22:37.616 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:37 smithi175 ceph-mon[101434]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 133'9196 2024-08-17T17:22:37.616 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2412145796' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:37.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:37.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:37.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:37.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:37.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:37.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:37.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:38.036 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:38.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:38.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:38.408 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:38 smithi175 ceph-mon[101434]: pgmap v385: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 7.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25740/342819 objects degraded (7.508%); 41 KiB/s, 122 objects/s recovering 2024-08-17T17:22:38.408 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:38 smithi175 ceph-mon[101434]: osdmap e137: 8 total, 8 up, 7 in 2024-08-17T17:22:38.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25740/342819 objects degraded (7.508%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:38.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[104516]: pgmap v385: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 7.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25740/342819 objects degraded (7.508%); 41 KiB/s, 122 objects/s recovering 2024-08-17T17:22:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[104516]: osdmap e137: 8 total, 8 up, 7 in 2024-08-17T17:22:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25740/342819 objects degraded (7.508%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[109246]: pgmap v385: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 7.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 25740/342819 objects degraded (7.508%); 41 KiB/s, 122 objects/s recovering 2024-08-17T17:22:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[109246]: osdmap e137: 8 total, 8 up, 7 in 2024-08-17T17:22:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25740/342819 objects degraded (7.508%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:38.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:38.568 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:22:38.568 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:22:38.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140794 5.e 5:70adc07f:::benchmark_data_smithi064_147470_object8850:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.346727522999998, 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.01103878, 2024-08-17T17:22:38.569 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.570 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.570 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.570 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.570 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.570 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140794 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217387+0000", 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217394+0000", 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8109999999999997e-06 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217408+0000", 2024-08-17T17:22:38.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3536999999999999e-05 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217415+0000", 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5680000000000002e-06 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217613+0000", 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019751099999999999 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217621+0000", 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4849999999999992e-06 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210300+0000", 2024-08-17T17:22:38.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.99267841499999998 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210423+0000", 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012385 2024-08-17T17:22:38.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210935+0000", 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00051183100000000003 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225429+0000", 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014493239999999999 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225526+0000", 2024-08-17T17:22:38.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7489000000000001e-05 2024-08-17T17:22:38.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228353+0000", 2024-08-17T17:22:38.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0028269509999999999 2024-08-17T17:22:38.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228388+0000", 2024-08-17T17:22:38.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5176000000000003e-05 2024-08-17T17:22:38.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228428+0000", 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9483999999999997e-05 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140802 5.1 5:8420ad0a:::benchmark_data_smithi064_147470_object8858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.281105487000001, 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93395603000000005, 2024-08-17T17:22:38.585 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140802 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.586 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283009+0000", 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283017+0000", 2024-08-17T17:22:38.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6709999999999999e-06 2024-08-17T17:22:38.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283032+0000", 2024-08-17T17:22:38.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5159e-05 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283063+0000", 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0406e-05 2024-08-17T17:22:38.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283108+0000", 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4935999999999998e-05 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283118+0000", 2024-08-17T17:22:38.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0757000000000001e-05 2024-08-17T17:22:38.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210288+0000", 2024-08-17T17:22:38.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.92716919799999997 2024-08-17T17:22:38.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210421+0000", 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013376499999999999 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210968+0000", 2024-08-17T17:22:38.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054626499999999999 2024-08-17T17:22:38.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216852+0000", 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058840200000000002 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216887+0000", 2024-08-17T17:22:38.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5522000000000003e-05 2024-08-17T17:22:38.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216894+0000", 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.63e-06 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216933+0000", 2024-08-17T17:22:38.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9208999999999997e-05 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216967+0000", 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3924000000000002e-05 2024-08-17T17:22:38.600 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140822 5.1 5:8accaffa:::benchmark_data_smithi064_147470_object8878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.601 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:22:38.602 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.186836768999999, 2024-08-17T17:22:38.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85167474700000001, 2024-08-17T17:22:38.602 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.602 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.602 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140822 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377278+0000", 2024-08-17T17:22:38.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:22:38.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377285+0000", 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377301+0000", 2024-08-17T17:22:38.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5423000000000001e-05 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377309+0000", 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0390000000000005e-06 2024-08-17T17:22:38.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377380+0000", 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0784999999999994e-05 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377389+0000", 2024-08-17T17:22:38.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1179999999999998e-06 2024-08-17T17:22:38.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216990+0000", 2024-08-17T17:22:38.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83960149699999997 2024-08-17T17:22:38.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217105+0000", 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114775 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217195+0000", 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9908999999999996e-05 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228788+0000", 2024-08-17T17:22:38.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011592966 2024-08-17T17:22:38.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228835+0000", 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6978000000000003e-05 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228873+0000", 2024-08-17T17:22:38.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7605000000000003e-05 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228921+0000", 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8013000000000003e-05 2024-08-17T17:22:38.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228954+0000", 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3775000000000001e-05 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.617 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.618 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.618 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140827 5.1 5:8b17e64e:::benchmark_data_smithi064_147470_object8883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.618 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.186790522999999, 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85178876800000003, 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.619 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140827 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:22:38.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377325+0000", 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377328+0000", 2024-08-17T17:22:38.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1370000000000002e-06 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377335+0000", 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8260000000000004e-06 2024-08-17T17:22:38.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377340+0000", 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1730000000000003e-06 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377409+0000", 2024-08-17T17:22:38.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8702999999999996e-05 2024-08-17T17:22:38.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377415+0000", 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8869999999999997e-06 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220644+0000", 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84322923900000002 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220744+0000", 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9375000000000003e-05 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220857+0000", 2024-08-17T17:22:38.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000113417 2024-08-17T17:22:38.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228842+0000", 2024-08-17T17:22:38.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079852320000000001 2024-08-17T17:22:38.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228994+0000", 2024-08-17T17:22:38.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001518 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229024+0000", 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0190999999999999e-05 2024-08-17T17:22:38.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229078+0000", 2024-08-17T17:22:38.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3427999999999997e-05 2024-08-17T17:22:38.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229115+0000", 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6875999999999997e-05 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140835 5.1 5:88614440:::benchmark_data_smithi064_147470_object8891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.178624033999998, 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85468788299999998, 2024-08-17T17:22:38.637 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140835 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385491+0000", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385500+0000", 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.738e-06 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385518+0000", 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8902e-05 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385527+0000", 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7320000000000004e-06 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385583+0000", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6035000000000001e-05 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385593+0000", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4089999999999992e-06 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224815+0000", 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83922198699999995 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224935+0000", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120415 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225062+0000", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000127467 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229173+0000", 2024-08-17T17:22:38.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041108539999999997 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235012+0000", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058387229999999997 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240111+0000", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0050989340000000003 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240148+0000", 2024-08-17T17:22:38.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6748000000000001e-05 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240180+0000", 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2509999999999999e-05 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140846 5.1 5:8aef7983:::benchmark_data_smithi064_147470_object8902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.644 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.161140466999999, 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84633296700000005, 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140846 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:22:38.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402975+0000", 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402981+0000", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.19e-06 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402993+0000", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1949e-05 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403001+0000", 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403040+0000", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9366000000000002e-05 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403047+0000", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3019999999999997e-06 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229200+0000", 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82615223199999999 2024-08-17T17:22:38.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229313+0000", 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011332699999999999 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229435+0000", 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012217400000000001 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240217+0000", 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010781689000000001 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240246+0000", 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9513999999999999e-05 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249221+0000", 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0089744300000000003 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249270+0000", 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9008999999999998e-05 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249309+0000", 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9031e-05 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140847 5.e 5:7ffdcb03:::benchmark_data_smithi064_147470_object8903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.159363299999999, 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82381208399999994, 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140847 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404752+0000", 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:22:38.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404758+0000", 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7039999999999999e-06 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404770+0000", 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2148e-05 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404777+0000", 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6889999999999998e-06 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404822+0000", 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4827000000000003e-05 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404854+0000", 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2366999999999997e-05 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218174+0000", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81331985799999995 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218278+0000", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010474799999999999 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218390+0000", 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111401 2024-08-17T17:22:38.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225460+0000", 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0070701369999999998 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225563+0000", 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010276 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228497+0000", 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002934613 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228535+0000", 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7539000000000001e-05 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228565+0000", 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0238000000000001e-05 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.658 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140853 5.1 5:8ec8c268:::benchmark_data_smithi064_147470_object8909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.157702050000001, 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.843158564, 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.659 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140853 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406413+0000", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406419+0000", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.057e-06 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406433+0000", 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4328000000000001e-05 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406440+0000", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8419999999999999e-06 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406489+0000", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9506999999999999e-05 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406498+0000", 2024-08-17T17:22:38.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3429999999999996e-06 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235037+0000", 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82853932900000005 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235131+0000", 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4499000000000004e-05 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235239+0000", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107617 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249349+0000", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014109981000000001 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249391+0000", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1950999999999999e-05 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249499+0000", 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010786899999999999 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249541+0000", 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249573+0000", 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2308999999999998e-05 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.665 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140856 5.e 5:7672cac1:::benchmark_data_smithi064_147470_object8912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.149866348, 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82466934800000002, 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.666 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140856 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.667 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414249+0000", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414255+0000", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414267+0000", 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1759e-05 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414272+0000", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2039999999999996e-06 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414318+0000", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6096e-05 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414326+0000", 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222146+0000", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.80782070699999997 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222239+0000", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2288999999999997e-05 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222337+0000", 2024-08-17T17:22:38.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8740000000000007e-05 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228879+0000", 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0065412170000000002 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.230356+0000", 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014777900000000001 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238835+0000", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0084788959999999997 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238881+0000", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5371000000000003e-05 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238919+0000", 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8642e-05 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.672 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140876 5.1 5:8e383c61:::benchmark_data_smithi064_147470_object8932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.112860106999999, 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79844427600000001, 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.673 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140876 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.674 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451255+0000", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451262+0000", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1709999999999996e-06 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451275+0000", 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2833000000000001e-05 2024-08-17T17:22:38.675 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451293+0000", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8309999999999999e-05 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451336+0000", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3052e-05 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451346+0000", 2024-08-17T17:22:38.676 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8879999999999992e-06 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243525+0000", 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79217860100000004 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243656+0000", 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013163500000000001 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.677 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243790+0000", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013325699999999999 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249472+0000", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0056827220000000003 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249604+0000", 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000131984 2024-08-17T17:22:38.678 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249646+0000", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2058999999999998e-05 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249675+0000", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8381000000000002e-05 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249701+0000", 2024-08-17T17:22:38.679 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5769e-05 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140877 5.e 5:70e8904e:::benchmark_data_smithi064_147470_object8933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.110642903999999, 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.78553511300000001, 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.680 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140877 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.681 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453472+0000", 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453480+0000", 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:22:38.682 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453490+0000", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0473e-05 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453497+0000", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0570000000000004e-06 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453537+0000", 2024-08-17T17:22:38.683 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9613000000000002e-05 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453544+0000", 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225585+0000", 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.77204120700000001 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.684 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225685+0000", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000100185 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225791+0000", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010613800000000001 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229184+0000", 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033933909999999999 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.685 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238791+0000", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096060130000000001 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238950+0000", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015964399999999999 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238981+0000", 2024-08-17T17:22:38.686 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1239999999999999e-05 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.239009+0000", 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7203e-05 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150244 5.f 5:ffa6cc92:::benchmark_data_smithi064_147470_object18300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:36.999903+0000", 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 1.564213168, 2024-08-17T17:22:38.687 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.14855689799999999, 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150244 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999903+0000", 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.688 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999903+0000", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999902+0000", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999910+0000", 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.689 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999928+0000", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8312999999999999e-05 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:36.999935+0000", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9269999999999999e-06 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.013605+0000", 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013669819 2024-08-17T17:22:38.690 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.013720+0000", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000115029 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.095036+0000", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.081315881000000007 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.095116+0000", 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0455000000000001e-05 2024-08-17T17:22:38.691 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.095255+0000", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013928399999999999 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.113513+0000", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.018257577000000001 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.123597+0000", 2024-08-17T17:22:38.692 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010083699 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.144230+0000", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.020633002000000001 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.144285+0000", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5090000000000003e-05 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.148460+0000", 2024-08-17T17:22:38.693 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041752639999999997 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150664 5.f 5:f7ffbecc:::benchmark_data_smithi064_147470_object18720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.870008907, 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24416275700000001, 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.694 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150664 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694106+0000", 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694113+0000", 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1339999999999998e-06 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.696 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694126+0000", 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2644e-05 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694132+0000", 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6150000000000002e-06 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.739879+0000", 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.045747115999999997 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.697 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.740006+0000", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012663900000000001 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.792997+0000", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.052990644000000003 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793094+0000", 2024-08-17T17:22:38.698 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7040000000000006e-05 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793282+0000", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018835799999999999 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804038+0000", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010755603000000001 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804047+0000", 2024-08-17T17:22:38.699 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9590000000000001e-06 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.934993+0000", 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.13094678300000001 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.935051+0000", 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8128999999999999e-05 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.700 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.938270+0000", 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032187499999999998 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150694 5.f 5:f212f019:::benchmark_data_smithi064_147470_object18750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.734639+0000", 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.82947706300000001, 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.192468905, 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.701 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150694 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734639+0000", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734639+0000", 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.702 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734639+0000", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734642+0000", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5669999999999999e-06 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734650+0000", 2024-08-17T17:22:38.703 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1359999999999999e-06 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.734655+0000", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4709999999999998e-06 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.770794+0000", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.036139488999999997 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.770964+0000", 2024-08-17T17:22:38.704 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000169574 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.771123+0000", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015906000000000001 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.784621+0000", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013497970999999999 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.792962+0000", 2024-08-17T17:22:38.705 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0083411920000000007 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.922585+0000", 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.12962299599999999 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.922619+0000", 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3377000000000002e-05 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.706 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.927108+0000", 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0044895919999999997 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150700 5.f 5:ff3e34ae:::benchmark_data_smithi064_147470_object18756:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.748876+0000", 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.81524037999999999, 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18207729, 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.707 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150700 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748876+0000", 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.708 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748876+0000", 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748874+0000", 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748881+0000", 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0559999999999999e-06 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.709 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748898+0000", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7234000000000001e-05 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.748907+0000", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0059999999999998e-06 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.775878+0000", 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.026970484999999999 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.710 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.776034+0000", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015564600000000001 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.776205+0000", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000171669 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.792969+0000", 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.016763402 2024-08-17T17:22:38.711 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.795983+0000", 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030143520000000001 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.927176+0000", 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.13119309500000001 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.712 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.927218+0000", 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1483000000000002e-05 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.930953+0000", 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037357509999999998 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150712 5.f 5:f837514f:::benchmark_data_smithi064_147470_object18768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.713 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.765410+0000", 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.79870615899999997, 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.16879955699999999, 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150712 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.714 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765410+0000", 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765410+0000", 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765409+0000", 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.715 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765415+0000", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2879999999999998e-06 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765437+0000", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1273000000000001e-05 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.765444+0000", 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8269999999999998e-06 2024-08-17T17:22:38.716 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.780879+0000", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015434194 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.781033+0000", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015468 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.781198+0000", 2024-08-17T17:22:38.717 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016500600000000001 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.796017+0000", 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014818970000000001 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803995+0000", 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079778620000000005 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.718 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.931008+0000", 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.12701341899999999 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.931059+0000", 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0807999999999999e-05 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.934210+0000", 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0031504789999999999 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.719 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150726 5.f 5:f296d3d5:::benchmark_data_smithi064_147470_object18782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.789146+0000", 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.77497044100000001, 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.15280877600000001, 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150726 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.720 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789146+0000", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789146+0000", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789145+0000", 2024-08-17T17:22:38.721 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789151+0000", 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6590000000000001e-06 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789164+0000", 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2578000000000001e-05 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.789170+0000", 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.800621+0000", 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011450952 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.800793+0000", 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017196099999999999 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.723 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.800977+0000", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018376500000000001 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804087+0000", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003109559 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.807305+0000", 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003218294 2024-08-17T17:22:38.724 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.938335+0000", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.131030221 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.938447+0000", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112229 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.941955+0000", 2024-08-17T17:22:38.725 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035074469999999999 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150737 5.f 5:f4de27c7:::benchmark_data_smithi064_147470_object18793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.803671+0000", 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.76044585600000003, 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.14278217900000001, 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.726 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150737 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803671+0000", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803671+0000", 2024-08-17T17:22:38.727 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803669+0000", 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803675+0000", 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6500000000000001e-06 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803686+0000", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0976e-05 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.803692+0000", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2369999999999997e-06 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804116+0000", 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00042433500000000002 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804233+0000", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011700400000000001 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804354+0000", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120788 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824184+0000", 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.019830029999999998 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.730 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824198+0000", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3777999999999999e-05 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.942025+0000", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.117827433 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.942090+0000", 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4560000000000005e-05 2024-08-17T17:22:38.731 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.946453+0000", 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043628820000000002 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150743 5.f 5:f8fc72c3:::benchmark_data_smithi064_147470_object18799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.75188392800000003, 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27548363100000001, 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.732 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150743 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.733 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:22:38.734 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.734 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.734 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.734 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.734 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812231+0000", 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812237+0000", 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.694e-06 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812248+0000", 2024-08-17T17:22:38.735 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1012000000000001e-05 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812253+0000", 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2719999999999997e-06 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824221+0000", 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011967396999999999 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.736 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824354+0000", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013296799999999999 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988177+0000", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.163823843 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988257+0000", 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9561000000000006e-05 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.737 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988353+0000", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5855999999999998e-05 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.004347+0000", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015994228999999999 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.012117+0000", 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0077704050000000002 2024-08-17T17:22:38.738 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083403+0000", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071285541999999993 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083488+0000", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5028e-05 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.087716+0000", 2024-08-17T17:22:38.739 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042280720000000003 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150839 5.f 5:fe36e86c:::benchmark_data_smithi064_147470_object18895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:38.000112+0000", 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.56400404400000004, 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.14022095600000001, 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:22:38.740 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150839 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000112+0000", 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000112+0000", 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.741 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000111+0000", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000118+0000", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0249999999999997e-06 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000134+0000", 2024-08-17T17:22:38.742 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5974e-05 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.000141+0000", 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4959999999999999e-06 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.004366+0000", 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042251779999999996 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:22:38.743 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.004441+0000", 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4388999999999995e-05 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.114286+0000", 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.10984474800000001 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.114405+0000", 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000119782 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.744 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.114562+0000", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000156577 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.135623+0000", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.021061425000000002 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.135673+0000", 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0000000000000002e-05 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.745 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.135682+0000", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9889999999999998e-06 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.135735+0000", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3176e-05 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.140333+0000", 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0045978199999999999 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.746 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.747 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.747 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:22:38.747 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:22:38.747 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:22:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:38.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.927 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:38.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:39.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:39.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:39.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:39.203 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:39.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.428 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:39.458 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:39.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:39 smithi064 ceph-mon[104516]: osdmap e138: 8 total, 8 up, 7 in 2024-08-17T17:22:39.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:39 smithi064 ceph-mon[109246]: osdmap e138: 8 total, 8 up, 7 in 2024-08-17T17:22:39.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:39.573 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:39 smithi175 ceph-mon[101434]: osdmap e138: 8 total, 8 up, 7 in 2024-08-17T17:22:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:39.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:39.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:39.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:39.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:39.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:39.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:39.997 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:40.036 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:22:40.037 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:22:40.037 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-08-17T17:22:40.037 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:22:40.037 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:22:40.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-08-17T17:22:40.099 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:40.122 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:40.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:40.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:40.227 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:22:40.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:40.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:40 smithi175 ceph-mon[101434]: pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33146/340923 objects degraded (9.722%); 6931/340923 objects misplaced (2.033%); 2.7 MiB/s, 394 objects/s recovering 2024-08-17T17:22:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:40 smithi175 ceph-mon[101434]: osdmap e139: 8 total, 8 up, 7 in 2024-08-17T17:22:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:40.428 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:40.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:22:40.530 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:40.548 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[104516]: pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33146/340923 objects degraded (9.722%); 6931/340923 objects misplaced (2.033%); 2.7 MiB/s, 394 objects/s recovering 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[104516]: osdmap e139: 8 total, 8 up, 7 in 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[109246]: pgmap v388: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 4 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33146/340923 objects degraded (9.722%); 6931/340923 objects misplaced (2.033%); 2.7 MiB/s, 394 objects/s recovering 2024-08-17T17:22:40.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[109246]: osdmap e139: 8 total, 8 up, 7 in 2024-08-17T17:22:40.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:40.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:40.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:40.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:40.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:40.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:40.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:40.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:40.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:41.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:41.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:41.199 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:41.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:41.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.360 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:41.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:41.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[104516]: osdmap e140: 8 total, 8 up, 7 in 2024-08-17T17:22:41.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:41.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:41.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:41 smithi064 ceph-mon[109246]: osdmap e140: 8 total, 8 up, 7 in 2024-08-17T17:22:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:41.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:41.518 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:41.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:41.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:41 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/862309055' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:41 smithi175 ceph-mon[101434]: osdmap e140: 8 total, 8 up, 7 in 2024-08-17T17:22:41.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:41.781 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:41.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.930 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:42.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:42.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:42.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:42.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:42.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:42.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:42.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:42.496 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:42.502 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:42 smithi175 ceph-mon[101434]: pgmap v391: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 33146/339702 objects degraded (9.757%); 20083/339702 objects misplaced (5.912%); 15 MiB/s, 443 objects/s recovering 2024-08-17T17:22:42.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:42 smithi064 ceph-mon[104516]: pgmap v391: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 33146/339702 objects degraded (9.757%); 20083/339702 objects misplaced (5.912%); 15 MiB/s, 443 objects/s recovering 2024-08-17T17:22:42.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:42 smithi064 ceph-mon[109246]: pgmap v391: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 33146/339702 objects degraded (9.757%); 20083/339702 objects misplaced (5.912%); 15 MiB/s, 443 objects/s recovering 2024-08-17T17:22:42.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:42.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:42.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:42.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:42.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:42.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:42.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:43.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:43.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:43.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:22:43.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:43.320 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:22:43.321 INFO:teuthology.orchestra.run.smithi175.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:22:43.321 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:22:43.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:43.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33146/339702 objects degraded (9.757%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33146/339702 objects degraded (9.757%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33146/339702 objects degraded (9.757%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:43.439 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.470 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.542 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:43.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:43.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:43.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.799 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:43.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:43.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.944 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:44.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:43] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-08-17T17:22:44.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:44.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:44.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:44.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:44.406 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:44 smithi175 ceph-mon[101434]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 33210/335067 objects degraded (9.911%); 33295/335067 objects misplaced (9.937%); 21 MiB/s, 486 objects/s recovering 2024-08-17T17:22:44.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:44.490 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:44.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:44.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.617 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:44.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:44 smithi064 ceph-mon[104516]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 33210/335067 objects degraded (9.911%); 33295/335067 objects misplaced (9.937%); 21 MiB/s, 486 objects/s recovering 2024-08-17T17:22:44.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:44 smithi064 ceph-mon[109246]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 33210/335067 objects degraded (9.911%); 33295/335067 objects misplaced (9.937%); 21 MiB/s, 486 objects/s recovering 2024-08-17T17:22:44.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:44.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:44.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:44.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:45.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:45.045 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:45.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:45.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:45.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:45.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:45.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:45.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.622 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:45.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:45.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:45.765 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:45.847 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:45.848 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:45.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.891 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:45.891 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:45.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:45.924 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:45.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:45.956 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:45.957 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:45.987 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:45.988 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:45.999 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:46.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:46.026 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:46.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:46.062 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:46.062 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:46.094 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:46.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:46.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:46.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:46.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:46.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:46.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:46.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:46.598 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:46 smithi175 ceph-mon[101434]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 33210/333648 objects degraded (9.954%); 33295/333648 objects misplaced (9.979%); 16 MiB/s, 350 objects/s recovering 2024-08-17T17:22:46.598 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3924554920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:46 smithi064 ceph-mon[104516]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 33210/333648 objects degraded (9.954%); 33295/333648 objects misplaced (9.979%); 16 MiB/s, 350 objects/s recovering 2024-08-17T17:22:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3924554920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:46 smithi064 ceph-mon[109246]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 33210/333648 objects degraded (9.954%); 33295/333648 objects misplaced (9.979%); 16 MiB/s, 350 objects/s recovering 2024-08-17T17:22:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3924554920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.732 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.748 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:46.748 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:46.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:47.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:47.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:47.231 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:47.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:47.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:47.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:47.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:47.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:47.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:47.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:47.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:47.822 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:47.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:47.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:48.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:48.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:48.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:48.227 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:22:48.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:48.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:48.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:48.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:48.518 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:48.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[104516]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 33287/327501 objects degraded (10.164%); 31888/327501 objects misplaced (9.737%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:22:48.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33287/327501 objects degraded (10.164%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:48.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[109246]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 33287/327501 objects degraded (10.164%); 31888/327501 objects misplaced (9.737%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:22:48.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33287/327501 objects degraded (10.164%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:48.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:48.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:48 smithi175 ceph-mon[101434]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 33287/327501 objects degraded (10.164%); 31888/327501 objects misplaced (9.737%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:22:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33287/327501 objects degraded (10.164%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:48.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:48.785 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:48.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:48.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:48.999 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:22:48.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:22:49.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:49.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:49.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:49.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:49.305 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:49.337 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:49.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:49.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:49.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:49.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:49.634 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:49.643 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:49.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:49.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:49.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:49.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:50.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:50.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:50.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:50.212 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:50.314 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:50.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:50 smithi175 ceph-mon[101434]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 33287/324657 objects degraded (10.253%); 30944/324657 objects misplaced (9.531%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:22:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3458463583' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:50.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:50.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:50.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:50.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[104516]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 33287/324657 objects degraded (10.253%); 30944/324657 objects misplaced (9.531%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3458463583' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[109246]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 33287/324657 objects degraded (10.253%); 30944/324657 objects misplaced (9.531%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3458463583' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:50.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:50.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:50.701 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:22:50.722 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:50.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:50.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:50.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:50.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.017 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:51.118 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:51.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:51.279 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:51.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:51.381 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:51.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:51.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:51.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:51.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:51.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:51.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:51.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[104516]: osdmap e141: 8 total, 8 up, 7 in 2024-08-17T17:22:51.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:51.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:51.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:51.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:51.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:51.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:51 smithi064 ceph-mon[109246]: osdmap e141: 8 total, 8 up, 7 in 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:51 smithi175 ceph-mon[101434]: osdmap e141: 8 total, 8 up, 7 in 2024-08-17T17:22:51.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:51.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:51.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:51.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:52.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:52.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:52.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:52.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:52.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:52.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:52.386 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:52.386 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:52 smithi175 ceph-mon[101434]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 33287/322953 objects degraded (10.307%); 29966/322953 objects misplaced (9.279%); 20 MiB/s, 329 objects/s recovering 2024-08-17T17:22:52.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:52.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:52.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:52.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:52 smithi064 ceph-mon[104516]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 33287/322953 objects degraded (10.307%); 29966/322953 objects misplaced (9.279%); 20 MiB/s, 329 objects/s recovering 2024-08-17T17:22:52.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:52 smithi064 ceph-mon[109246]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 33287/322953 objects degraded (10.307%); 29966/322953 objects misplaced (9.279%); 20 MiB/s, 329 objects/s recovering 2024-08-17T17:22:52.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:52.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:52.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:52.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:53.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:53.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:53.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:53.127 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:22:53.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:53.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:53.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:53.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:53.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:53.407 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:53.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33287/322953 objects degraded (10.307%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:53.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:53.485 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:53.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:53.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33287/322953 objects degraded (10.307%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33287/322953 objects degraded (10.307%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:53.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:53.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:53.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:53.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:53.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:54.036 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:54.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:54.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:54.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:54.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:54.317 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:22:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:22:53] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-08-17T17:22:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:54.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:54.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:54.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:54.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:54.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:54.651 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:54 smithi175 ceph-mon[101434]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33380/317385 objects degraded (10.517%); 28806/317385 objects misplaced (9.076%); 19 MiB/s, 309 objects/s recovering 2024-08-17T17:22:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:54 smithi064 ceph-mon[104516]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33380/317385 objects degraded (10.517%); 28806/317385 objects misplaced (9.076%); 19 MiB/s, 309 objects/s recovering 2024-08-17T17:22:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:54 smithi064 ceph-mon[109246]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33380/317385 objects degraded (10.517%); 28806/317385 objects misplaced (9.076%); 19 MiB/s, 309 objects/s recovering 2024-08-17T17:22:54.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:54.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:54.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:54.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:54.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:55.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:55.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:55.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:55.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:55.287 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:55.287 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:55.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:55.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:55.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:55.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:55.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:55.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:55.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:55.832 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.866 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:55.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:55.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:22:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:56.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:56.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:56.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:56 smithi175 ceph-mon[101434]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33380/315741 objects degraded (10.572%); 28806/315741 objects misplaced (9.123%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:22:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:56.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:56.273 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:22:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:56.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:56.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[109246]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33380/315741 objects degraded (10.572%); 28806/315741 objects misplaced (9.123%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:22:56.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:56.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:56.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[104516]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 33380/315741 objects degraded (10.572%); 28806/315741 objects misplaced (9.123%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:22:56.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:22:56.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:56.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:56.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:56.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:56.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:56.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:56.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:56.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.840 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:56.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:56.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:56.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:57.054 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:22:57.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:57.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:57.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:57.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:57.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:57.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:57.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:57.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:57.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:57.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:22:57.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:57.535 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:57.637 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:57.665 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:57.768 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:57.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:57.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:57.922 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:57.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:22:58.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:58.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[104516]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 33503/309522 objects degraded (10.824%); 27445/309522 objects misplaced (8.867%); 19 MiB/s, 299 objects/s recovering 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[104516]: osdmap e142: 8 total, 8 up, 7 in 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[109246]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 33503/309522 objects degraded (10.824%); 27445/309522 objects misplaced (8.867%); 19 MiB/s, 299 objects/s recovering 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:58 smithi064 ceph-mon[109246]: osdmap e142: 8 total, 8 up, 7 in 2024-08-17T17:22:58.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:58.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:58.360 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:58.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:58 smithi175 ceph-mon[101434]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 33503/309522 objects degraded (10.824%); 27445/309522 objects misplaced (8.867%); 19 MiB/s, 299 objects/s recovering 2024-08-17T17:22:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:58 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2952750022' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:58 smithi175 ceph-mon[101434]: osdmap e142: 8 total, 8 up, 7 in 2024-08-17T17:22:58.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:58.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:58.591 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:58.592 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:22:58.592 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:58.605 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:58.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:58.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:22:58.644 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:58.677 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:22:58.678 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:58.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:22:58.710 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:58.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:58.754 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:22:58.754 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:58.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:58.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:58.793 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:22:58.794 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:58.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:22:58.825 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:58.862 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:22:58.862 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:58.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:58.895 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:58.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.033 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:59.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:59.134 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:59.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:59.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:59.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:59.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:59 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33503/309522 objects degraded (10.824%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:22:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2133548442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:59 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33503/309522 objects degraded (10.824%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:22:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2133548442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:59.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:59 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33503/309522 objects degraded (10.824%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:59.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:22:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2133548442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:59.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:59.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:59.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:59.689 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:59.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:59.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:22:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:59.931 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:59.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:22:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:00.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:00.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:00.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:00.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:00.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:00.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:00.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:00 smithi175 ceph-mon[101434]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 33503/306873 objects degraded (10.918%); 26690/306873 objects misplaced (8.697%); 22 MiB/s, 350 objects/s recovering 2024-08-17T17:23:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:00 smithi064 ceph-mon[104516]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 33503/306873 objects degraded (10.918%); 26690/306873 objects misplaced (8.697%); 22 MiB/s, 350 objects/s recovering 2024-08-17T17:23:00.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:00 smithi064 ceph-mon[109246]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 33503/306873 objects degraded (10.918%); 26690/306873 objects misplaced (8.697%); 22 MiB/s, 350 objects/s recovering 2024-08-17T17:23:00.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:00.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:00.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:00.655 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:00.757 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:00.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:00.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:00.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:00.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:01.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:01.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:01.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:01.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:01.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:01.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:01.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:01.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:01.525 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:01.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:01.627 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:01.641 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:23:01.642 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:23:01.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.642 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.34626.0:164241 5.3 e142/135 5:c81d4a50:::benchmark_data_smithi064_147470_object32297:head v 142'10265, mlcod=142'10263)", 2024-08-17T17:23:01.642 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:01.632713+0000", 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.004571571, 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0046131699999999998, 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:01.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632713+0000", 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632713+0000", 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:01.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632712+0000", 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632722+0000", 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5799999999999998e-06 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632737+0000", 2024-08-17T17:23:01.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5472999999999999e-05 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632746+0000", 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5210000000000002e-06 2024-08-17T17:23:01.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632804+0000", 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8638000000000003e-05 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:01.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:01.632836+0000", 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2132999999999999e-05 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:01.650 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:01.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:01.651 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:23:01.651 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 1 2024-08-17T17:23:01.651 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:23:01.674 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:01.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:01.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:01.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:01.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:02.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:02.099 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:02.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:02.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:02.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:02.353 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:02 smithi175 ceph-mon[101434]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33503/305286 objects degraded (10.974%); 25836/305286 objects misplaced (8.463%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:23:02.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:23:02.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:02 smithi064 ceph-mon[104516]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33503/305286 objects degraded (10.974%); 25836/305286 objects misplaced (8.463%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:23:02.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:02 smithi064 ceph-mon[109246]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 33503/305286 objects degraded (10.974%); 25836/305286 objects misplaced (8.463%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:23:02.467 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:02.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:02.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:02.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:02.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:02.672 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:02.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:02.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:02.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:02.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:02.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:03.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:03.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:03.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:03.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:03.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:03.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:03.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33503/305286 objects degraded (10.974%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:03.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33503/305286 objects degraded (10.974%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:03.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33503/305286 objects degraded (10.974%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:03.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:03.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:03.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:03.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:03.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.772 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:03.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:03.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:03.920 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:04.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:04.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:04.098 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:03] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-08-17T17:23:04.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.190 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:04.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:04.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:04.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:04.374 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:04 smithi064 ceph-mon[109246]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 33611/300021 objects degraded (11.203%); 24659/300021 objects misplaced (8.219%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:04.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:04 smithi064 ceph-mon[104516]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 33611/300021 objects degraded (11.203%); 24659/300021 objects misplaced (8.219%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:04 smithi175 ceph-mon[101434]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 33611/300021 objects degraded (11.203%); 24659/300021 objects misplaced (8.219%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:04.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:04.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:04.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:04.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:04.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:04.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:04.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:04.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:04.907 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:23:04.908 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:23:04.908 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:23:04.908 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:23:04.908 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:23:05.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:05.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:05.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:05.105 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:23:05.112 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:23:05.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:05.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:23:05.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4215630537' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:05 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.185 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:05.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:05.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:05.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:05.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:23:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4215630537' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:23:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4215630537' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:05 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:05.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:05.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:05.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:23:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:05.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:05.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:05.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:05.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:05.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:05.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:23:05.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:05.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:05.952 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:05.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:06.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:06.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:06.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:06.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:06.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:06.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:06 smithi175 ceph-mon[101434]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 33611/298452 objects degraded (11.262%); 24659/298452 objects misplaced (8.262%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:06 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:06 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:23:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:06 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:06 smithi175 ceph-mon[101434]: osdmap e143: 8 total, 8 up, 7 in 2024-08-17T17:23:06.416 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:06.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.438 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:06.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[109246]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 33611/298452 objects degraded (11.262%); 24659/298452 objects misplaced (8.262%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[109246]: osdmap e143: 8 total, 8 up, 7 in 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[104516]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 33611/298452 objects degraded (11.262%); 24659/298452 objects misplaced (8.262%); 18 MiB/s, 280 objects/s recovering 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2235017032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:23:06.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:06.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:06 smithi064 ceph-mon[104516]: osdmap e143: 8 total, 8 up, 7 in 2024-08-17T17:23:06.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:06.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:06.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:06.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:06.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:06.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:06.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:06.967 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:06.978 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:07.010 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:07.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:07.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:07.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:07.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:07.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:07.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:07.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:07.820 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.854 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:07.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:07.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.999 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:07.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:08.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:08.140 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:08.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:08.242 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:08.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:08.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:08 smithi175 ceph-mon[101434]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 33731/295020 objects degraded (11.433%); 23351/295020 objects misplaced (7.915%); 18 MiB/s, 286 objects/s recovering 2024-08-17T17:23:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33731/295020 objects degraded (11.433%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:08.419 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:08.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:08 smithi064 ceph-mon[104516]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 33731/295020 objects degraded (11.433%); 23351/295020 objects misplaced (7.915%); 18 MiB/s, 286 objects/s recovering 2024-08-17T17:23:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33731/295020 objects degraded (11.433%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:08 smithi064 ceph-mon[109246]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 496 op/s; 33731/295020 objects degraded (11.433%); 23351/295020 objects misplaced (7.915%); 18 MiB/s, 286 objects/s recovering 2024-08-17T17:23:08.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33731/295020 objects degraded (11.433%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:15441 5.2 5:4d04414a:::benchmark_data_smithi064_147470_object15440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:23:08.516 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 150.21740401700001, 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.101536945, 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 15441 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295286+0000", 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295359+0000", 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2199000000000004e-05 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295370+0000", 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1088e-05 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295377+0000", 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8519999999999998e-06 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295427+0000", 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0229999999999998e-05 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295563+0000", 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000136332 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295712+0000", 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014863800000000001 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.296447+0000", 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00073474800000000002 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.296811+0000", 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036494799999999998 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396739+0000", 2024-08-17T17:23:08.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.099927929999999998 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396788+0000", 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8658000000000001e-05 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396825+0000", 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6554000000000003e-05 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.523 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:56631 5.2 5:45da10c3:::benchmark_data_smithi064_147470_object56630:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:20:57.568911+0000", 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 130.94378113900001, 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058667919999999998, 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 56631 2024-08-17T17:23:08.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568911+0000", 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568911+0000", 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568909+0000", 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568975+0000", 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6338000000000001e-05 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568992+0000", 2024-08-17T17:23:08.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6217000000000001e-05 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.568998+0000", 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3049999999999998e-06 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.569037+0000", 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8788999999999997e-05 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.569132+0000", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5370000000000003e-05 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.569265+0000", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013258700000000001 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.570764+0000", 2024-08-17T17:23:08.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014991850000000001 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.574153+0000", 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033894789999999999 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.627466+0000", 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.053312760000000001 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.627521+0000", 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5164000000000002e-05 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.627578+0000", 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7302999999999997e-05 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.530 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:56690 5.2 5:4690ef43:::benchmark_data_smithi064_147470_object56689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 130.84594562800001, 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.10647835, 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.531 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:08.532 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.532 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 56690 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666744+0000", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666809+0000", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5030000000000006e-05 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666832+0000", 2024-08-17T17:23:08.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2864999999999999e-05 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666840+0000", 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8669999999999994e-06 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666889+0000", 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8934000000000003e-05 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.667015+0000", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012589900000000001 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.667157+0000", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014245699999999999 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.668637+0000", 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001479749 2024-08-17T17:23:08.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.669409+0000", 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00077196899999999995 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773138+0000", 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.103729397 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773187+0000", 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8717999999999997e-05 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773224+0000", 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7245e-05 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:67647 5.2 5:4c74f656:::benchmark_data_smithi064_147470_object67646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:21:04.738155+0000", 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.774537146, 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.059744160999999997, 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 67647 2024-08-17T17:23:08.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738155+0000", 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738155+0000", 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738153+0000", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738229+0000", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5625999999999997e-05 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738248+0000", 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9278000000000001e-05 2024-08-17T17:23:08.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.738256+0000", 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6850000000000001e-06 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.739060+0000", 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00080383199999999998 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.739194+0000", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013460999999999999 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.739334+0000", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013945799999999999 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.741423+0000", 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0020897289999999998 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.742335+0000", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00091128400000000005 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.797695+0000", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.055360197 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.797773+0000", 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8128999999999997e-05 2024-08-17T17:23:08.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:04.797899+0000", 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012570300000000001 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:73634 5.2 5:428715b5:::benchmark_data_smithi064_147470_object73633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:21:08.913036+0000", 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 119.599655764, 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.056903439, 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 73634 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913036+0000", 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913036+0000", 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913034+0000", 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913072+0000", 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8166e-05 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913091+0000", 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8508e-05 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913100+0000", 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7180000000000002e-06 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913271+0000", 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017122099999999999 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.913629+0000", 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00035761300000000001 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.914115+0000", 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00048622999999999998 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.916524+0000", 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0024088709999999999 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.916944+0000", 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000420698 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.969841+0000", 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.052897025 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.969894+0000", 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2544999999999999e-05 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:21:08.969939+0000", 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5433999999999999e-05 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.552 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:132248 5.2 5:4043f86e:::benchmark_data_smithi064_147470_object304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:05.401470+0000", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 63.111221461, 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.059910024999999999, 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 132248 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401470+0000", 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401470+0000", 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401468+0000", 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401478+0000", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0372e-05 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401498+0000", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9748000000000001e-05 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401507+0000", 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7900000000000005e-06 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401557+0000", 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0077000000000002e-05 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401674+0000", 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011699 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.401797+0000", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012282499999999999 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.405330+0000", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035331659999999999 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.405839+0000", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00050930500000000004 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.461320+0000", 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.055480868000000003 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.461354+0000", 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4017999999999999e-05 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.461380+0000", 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6095e-05 2024-08-17T17:23:08.558 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:132257 5.2 5:4c68c9f0:::benchmark_data_smithi064_147470_object313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 63.091303605, 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.075403524999999999, 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 132257 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421387+0000", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421392+0000", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.955e-06 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421403+0000", 2024-08-17T17:23:08.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0524e-05 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421409+0000", 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4779999999999996e-06 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421478+0000", 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8938999999999998e-05 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421608+0000", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012981900000000001 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421724+0000", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000116679 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.427301+0000", 2024-08-17T17:23:08.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0055767539999999997 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.427368+0000", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6888000000000001e-05 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496703+0000", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069335205999999996 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496749+0000", 2024-08-17T17:23:08.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5312999999999998e-05 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496792+0000", 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2991999999999998e-05 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140069 5.2 5:495968f6:::benchmark_data_smithi064_147470_object8125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.892563+0000", 2024-08-17T17:23:08.565 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 49.620128499000003, 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.065917464999999995, 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.566 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140069 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892563+0000", 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892563+0000", 2024-08-17T17:23:08.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892562+0000", 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892570+0000", 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5839999999999995e-06 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892584+0000", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3706999999999999e-05 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892592+0000", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7200000000000006e-06 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892628+0000", 2024-08-17T17:23:08.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6115000000000002e-05 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892704+0000", 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5703000000000003e-05 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.892768+0000", 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4626000000000001e-05 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.896646+0000", 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038773560000000002 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.896807+0000", 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016087100000000001 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.958392+0000", 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.061585945000000003 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.958439+0000", 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6829999999999997e-05 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.958481+0000", 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1335999999999999e-05 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140080 5.2 5:4c96634f:::benchmark_data_smithi064_147470_object8136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 49.604071980999997, 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071440704999999993, 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.573 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140080 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908619+0000", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908624+0000", 2024-08-17T17:23:08.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908635+0000", 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0871e-05 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908640+0000", 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4759999999999997e-06 2024-08-17T17:23:08.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908678+0000", 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7441999999999998e-05 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908770+0000", 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2101000000000005e-05 2024-08-17T17:23:08.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908860+0000", 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9926999999999995e-05 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.912365+0000", 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003505317 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.912701+0000", 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00033640999999999999 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.979959+0000", 2024-08-17T17:23:08.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067257586999999994 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.980015+0000", 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6258000000000002e-05 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.980060+0000", 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5207999999999998e-05 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140104 5.2 5:445a3862:::benchmark_data_smithi064_147470_object8160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 49.550094495000003, 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067168221, 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140104 2024-08-17T17:23:08.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962596+0000", 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962606+0000", 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0079e-05 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962624+0000", 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8082999999999999e-05 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962632+0000", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3939999999999996e-06 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964115+0000", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001482145 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964220+0000", 2024-08-17T17:23:08.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105437 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964332+0000", 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011228 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.969804+0000", 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.005471854 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.970808+0000", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00100413 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029688+0000", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058880035999999997 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029725+0000", 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6606000000000002e-05 2024-08-17T17:23:08.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029765+0000", 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0444000000000001e-05 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140105 5.2 5:46f7af23:::benchmark_data_smithi064_147470_object8161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.963843+0000", 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 49.548848546000002, 2024-08-17T17:23:08.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.066040331999999993, 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140105 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963843+0000", 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963843+0000", 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963842+0000", 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963850+0000", 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8609999999999998e-06 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963861+0000", 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0679000000000001e-05 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.963868+0000", 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8859999999999998e-06 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.967268+0000", 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003400003 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.967367+0000", 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9825000000000001e-05 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.967441+0000", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3158000000000007e-05 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.969929+0000", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0024884130000000001 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.970867+0000", 2024-08-17T17:23:08.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00093798500000000001 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029813+0000", 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058945722999999998 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029855+0000", 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2524000000000001e-05 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029883+0000", 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8291999999999999e-05 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140356 5.2 5:4d794467:::benchmark_data_smithi064_147470_object8412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 49.085011444999999, 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.073559667999999995, 2024-08-17T17:23:08.595 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140356 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427679+0000", 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427687+0000", 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1550000000000007e-06 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427697+0000", 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8770000000000005e-06 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427704+0000", 2024-08-17T17:23:08.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1350000000000003e-06 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427735+0000", 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0335999999999999e-05 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427823+0000", 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8755e-05 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427913+0000", 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9384999999999999e-05 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.431013+0000", 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030998760000000001 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.431694+0000", 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068099499999999999 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501141+0000", 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069447735999999996 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501193+0000", 2024-08-17T17:23:08.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1779e-05 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501240+0000", 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6662999999999997e-05 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140489 5.2 5:4dfa23d8:::benchmark_data_smithi064_147470_object8545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.602 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.828100577999997, 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.078668141999999996, 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140489 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684590+0000", 2024-08-17T17:23:08.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684599+0000", 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9120000000000004e-06 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684611+0000", 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2181000000000001e-05 2024-08-17T17:23:08.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684619+0000", 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5719999999999997e-06 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684656+0000", 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7302999999999998e-05 2024-08-17T17:23:08.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684739+0000", 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2908e-05 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684813+0000", 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4202999999999994e-05 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.688043+0000", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032301790000000001 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.688172+0000", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000128811 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763170+0000", 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.074998193000000005 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763216+0000", 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6176e-05 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763259+0000", 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2843999999999998e-05 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140767 5.2 5:4475c63a:::benchmark_data_smithi064_147470_object8823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.609 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.359759799000003, 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.085237967999999997, 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140767 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152931+0000", 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152937+0000", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3400000000000003e-06 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152949+0000", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1918e-05 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152955+0000", 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4420000000000003e-06 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152998+0000", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3167999999999997e-05 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.153095+0000", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6910999999999994e-05 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.153174+0000", 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8838999999999995e-05 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.156443+0000", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032685489999999999 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.157288+0000", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00084548399999999997 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238061+0000", 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.080772565000000004 2024-08-17T17:23:08.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238120+0000", 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.914e-05 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238170+0000", 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9869e-05 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140772 5.2 5:4d35615e:::benchmark_data_smithi064_147470_object8828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.351101999000001, 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.085906341999999997, 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140772 2024-08-17T17:23:08.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161579+0000", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.99999 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161601+0000", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1097000000000001e-05 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161635+0000", 2024-08-17T17:23:08.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4703000000000003e-05 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161655+0000", 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9422000000000001e-05 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161780+0000", 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000124845 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161978+0000", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019837000000000001 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.162114+0000", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000136394 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.165681+0000", 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035668789999999998 2024-08-17T17:23:08.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.165786+0000", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105247 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247377+0000", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.081591061000000006 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247448+0000", 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0835999999999997e-05 2024-08-17T17:23:08.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247496+0000", 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7707999999999998e-05 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140780 5.2 5:4421a3fd:::benchmark_data_smithi064_147470_object8836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:23:08.622 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.332373744999998, 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067303092999999994, 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140780 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180317+0000", 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180323+0000", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0109999999999997e-06 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180337+0000", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4303e-05 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180346+0000", 2024-08-17T17:23:08.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8599999999999999e-06 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180389+0000", 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3778e-05 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180521+0000", 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013125699999999999 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180653+0000", 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013204699999999999 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.184422+0000", 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037690570000000001 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.184455+0000", 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3497000000000001e-05 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247540+0000", 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.063084514999999994 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247585+0000", 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5453e-05 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247621+0000", 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5731000000000001e-05 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:143290 5.2 5:4a5cfda6:::benchmark_data_smithi064_147470_object11346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:24.857348+0000", 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 43.655343887999997, 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.064566284000000002, 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.629 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 143290 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857348+0000", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857348+0000", 2024-08-17T17:23:08.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857347+0000", 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857352+0000", 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6569999999999996e-06 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857361+0000", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9539999999999993e-06 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.857367+0000", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5119999999999997e-06 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.861116+0000", 2024-08-17T17:23:08.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037486049999999999 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.861250+0000", 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000133874 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.861361+0000", 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011115700000000001 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.865113+0000", 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037523579999999999 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.869331+0000", 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042184509999999998 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.921614+0000", 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.052282566000000003 2024-08-17T17:23:08.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.921786+0000", 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017192499999999999 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:24.921914+0000", 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012809900000000001 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:154468 5.2 5:4d402739:::benchmark_data_smithi064_147470_object22524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:44.801460+0000", 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 23.711231992999998, 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.054615828999999998, 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.636 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 154468 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801460+0000", 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801460+0000", 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801458+0000", 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801467+0000", 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7490000000000004e-06 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801485+0000", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7867999999999999e-05 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801494+0000", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8289999999999997e-06 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801542+0000", 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8470999999999997e-05 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801641+0000", 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8504000000000004e-05 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.801731+0000", 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0487000000000003e-05 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.805090+0000", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033589739999999998 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.805153+0000", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3150000000000004e-05 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.806274+0000", 2024-08-17T17:23:08.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001121227 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.855938+0000", 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.049663242000000003 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.855974+0000", 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6464000000000002e-05 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:44.856076+0000", 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101368 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:163139 5.2 5:41aac34f:::benchmark_data_smithi064_147470_object31195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.8226626120000002, 2024-08-17T17:23:08.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.096410124, 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 163139 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690027+0000", 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:23:08.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690037+0000", 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0297e-05 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690057+0000", 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0217999999999999e-05 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690067+0000", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8689999999999999e-06 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690120+0000", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3114e-05 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690229+0000", 2024-08-17T17:23:08.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010832699999999999 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690325+0000", 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6348000000000007e-05 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.693907+0000", 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035820880000000002 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.694092+0000", 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000184458 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.694644+0000", 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000552387 2024-08-17T17:23:08.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786314+0000", 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.091670351999999997 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786364+0000", 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9913000000000001e-05 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786439+0000", 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4970999999999998e-05 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:163379 5.2 5:406d3faa:::benchmark_data_smithi064_147470_object31435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:00.175275+0000", 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.337416589, 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.055818395, 2024-08-17T17:23:08.651 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 163379 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175275+0000", 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175275+0000", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175273+0000", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175285+0000", 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1501999999999999e-05 2024-08-17T17:23:08.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175306+0000", 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1172e-05 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175317+0000", 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0925e-05 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:23:08.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175376+0000", 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9215999999999997e-05 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.175491+0000", 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011552099999999999 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:23:08.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.176035+0000", 2024-08-17T17:23:08.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054311599999999995 2024-08-17T17:23:08.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.179398+0000", 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033634609999999999 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.179916+0000", 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000517648 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.197103+0000", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.017187332999999999 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.230910+0000", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.033806972999999997 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.230968+0000", 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8310999999999998e-05 2024-08-17T17:23:08.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.231093+0000", 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000125239 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:23:08.659 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:23:08.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.660 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:08.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:08.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:08.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:08.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:08.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:23:08.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:08.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:09.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:09.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:09.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:09.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:09.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:23:09.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.497 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:09.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:09.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:09.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:09.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:09.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:09.783 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:09.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:09.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:09.971 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:10.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:10.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:10.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:10.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:10.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:10.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:10.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:10 smithi175 ceph-mon[101434]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33731/289416 objects degraded (11.655%); 22659/289416 objects misplaced (7.829%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:23:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:10.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[104516]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33731/289416 objects degraded (11.655%); 22659/289416 objects misplaced (7.829%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:23:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[109246]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 33731/289416 objects degraded (11.655%); 22659/289416 objects misplaced (7.829%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:23:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:10.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:10.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:10.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:10.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:10.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:10.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:10.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:10.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:10.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:11.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.148 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:23:11.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:11.198 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:11.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:11.311 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:11 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3460842260' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.311 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:11 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3460842260' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:11.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:11 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3460842260' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:23:11.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.548 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:23:11.548 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:11.589 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:23:11.589 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:11.599 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:11.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:11.636 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:23:11.636 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:11.672 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:23:11.672 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:11.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:23:11.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:11.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.750 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:23:11.750 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.788 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:23:11.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:11.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:23:11.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:11.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:11.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:12.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:12.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:12.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:12.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:23:12.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:12.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:12 smithi175 ceph-mon[101434]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 33731/288927 objects degraded (11.675%); 22496/288927 objects misplaced (7.786%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:23:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:12 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:23:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:12 smithi175 ceph-mon[101434]: osdmap e144: 8 total, 8 up, 7 in 2024-08-17T17:23:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1923107661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:12.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[104516]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 33731/288927 objects degraded (11.675%); 22496/288927 objects misplaced (7.786%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:23:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:23:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[104516]: osdmap e144: 8 total, 8 up, 7 in 2024-08-17T17:23:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1923107661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[109246]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 33731/288927 objects degraded (11.675%); 22496/288927 objects misplaced (7.786%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:23:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:23:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[109246]: osdmap e144: 8 total, 8 up, 7 in 2024-08-17T17:23:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1923107661' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:12.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:12.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:12.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:12.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:12.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:12.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:13.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:13.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:13.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:13.198 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:13.198 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:13.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:13.337 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:13.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33731/288927 objects degraded (11.675%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33731/288927 objects degraded (11.675%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:13.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33731/288927 objects degraded (11.675%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:13.439 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:13.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:13.571 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:13.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:13.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:13.731 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:13.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:13.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:13.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:23:13.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:14.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:14.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:14.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:14.172 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:13] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.51.0" 2024-08-17T17:23:14.200 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:14.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:14.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:14.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:14.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:14.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:14 smithi064 ceph-mon[104516]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 33858/282474 objects degraded (11.986%); 20749/282474 objects misplaced (7.345%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:14 smithi064 ceph-mon[109246]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 33858/282474 objects degraded (11.986%); 20749/282474 objects misplaced (7.345%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:14.462 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:14 smithi175 ceph-mon[101434]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 33858/282474 objects degraded (11.986%); 20749/282474 objects misplaced (7.345%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:14.475 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:14.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.597 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:14.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:14.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:14.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:14.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:14.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:14.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:15.012 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:15.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:15.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:15.178 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:23:15.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:15.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:15.375 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1314122018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.375 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:15 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:15.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1314122018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:15 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1314122018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:15 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:15.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:15.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:15.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:15.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:15.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:15.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:15.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.957 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:15.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:16.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:16.061 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:16.061 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:16.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:16.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:16.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:16.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:16.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[104516]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33858/280773 objects degraded (12.059%); 20749/280773 objects misplaced (7.390%); 17 MiB/s, 276 objects/s recovering 2024-08-17T17:23:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[104516]: osdmap e145: 8 total, 8 up, 7 in 2024-08-17T17:23:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[109246]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33858/280773 objects degraded (12.059%); 20749/280773 objects misplaced (7.390%); 17 MiB/s, 276 objects/s recovering 2024-08-17T17:23:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:16 smithi064 ceph-mon[109246]: osdmap e145: 8 total, 8 up, 7 in 2024-08-17T17:23:16.443 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:16 smithi175 ceph-mon[101434]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33858/280773 objects degraded (12.059%); 20749/280773 objects misplaced (7.390%); 17 MiB/s, 276 objects/s recovering 2024-08-17T17:23:16.443 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:16 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:16.443 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:16.443 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:16 smithi175 ceph-mon[101434]: osdmap e145: 8 total, 8 up, 7 in 2024-08-17T17:23:16.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:16.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:16.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:16.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:16.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:16.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:16.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:23:16.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:16.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:17.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:17.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:17.122 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:17.275 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:17.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:17.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:17.442 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:17.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:17.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:17.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:17.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:18.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:18.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:18.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:18.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:18.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:18.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:18.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:18 smithi064 ceph-mon[104516]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 33955/277671 objects degraded (12.229%); 19570/277671 objects misplaced (7.048%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:23:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33955/277671 objects degraded (12.229%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:18 smithi064 ceph-mon[109246]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 33955/277671 objects degraded (12.229%); 19570/277671 objects misplaced (7.048%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:23:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33955/277671 objects degraded (12.229%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:18.559 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:18 smithi175 ceph-mon[101434]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 33955/277671 objects degraded (12.229%); 19570/277671 objects misplaced (7.048%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:23:18.559 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33955/277671 objects degraded (12.229%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:18.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:18.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:18.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:18.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:18.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:18.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.872 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:18.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:18.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:23:18.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:19.075 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:19.075 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:19.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:19.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:19.218 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:23:19.260 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:19.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:19.420 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:19.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:23:19.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:23:19.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:19.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:19.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:19.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:23:19.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:19.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:19.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:23:19.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:19.938 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:19.971 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:20.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:20.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:20.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:20.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:23:20.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:20.234 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:20.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:20.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:20.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[104516]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 33955/272076 objects degraded (12.480%); 18877/272076 objects misplaced (6.938%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:20.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:23:20.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[104516]: osdmap e146: 8 total, 8 up, 7 in 2024-08-17T17:23:20.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[109246]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 33955/272076 objects degraded (12.480%); 18877/272076 objects misplaced (6.938%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:20.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:23:20.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:20 smithi064 ceph-mon[109246]: osdmap e146: 8 total, 8 up, 7 in 2024-08-17T17:23:20.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:20.515 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:20.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:20 smithi175 ceph-mon[101434]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 33955/272076 objects degraded (12.480%); 18877/272076 objects misplaced (6.938%); 21 MiB/s, 334 objects/s recovering 2024-08-17T17:23:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:20 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/682230180' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:23:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:20 smithi175 ceph-mon[101434]: osdmap e146: 8 total, 8 up, 7 in 2024-08-17T17:23:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:20.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:20.749 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:20.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:20.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:23:20.798 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:20.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:20.826 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:20.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:23:20.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:20.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:23:20.871 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:20.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:20.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:23:20.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:20.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:23:20.941 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:20.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:20.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:23:20.972 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:21.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:23:21.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:21.036 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:23:21.037 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:21.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:21.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:21.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:21.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:21.308 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:21.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:21.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:21.472 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.509 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:21.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:21.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/189622939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:21.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/189622939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:21.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:21.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:21.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/189622939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:21.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:21.800 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:21.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:21.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:22.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:22.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:22.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:22.193 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:22.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:22.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:22.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:22.444 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:22.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:22.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:22.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:22.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:22 smithi175 ceph-mon[101434]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 33955/271518 objects degraded (12.506%); 18692/271518 objects misplaced (6.884%); 11 MiB/s, 171 objects/s recovering 2024-08-17T17:23:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:22 smithi064 ceph-mon[104516]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 33955/271518 objects degraded (12.506%); 18692/271518 objects misplaced (6.884%); 11 MiB/s, 171 objects/s recovering 2024-08-17T17:23:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:22 smithi064 ceph-mon[109246]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 33955/271518 objects degraded (12.506%); 18692/271518 objects misplaced (6.884%); 11 MiB/s, 171 objects/s recovering 2024-08-17T17:23:22.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:22.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.886 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.120 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:23.144 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:23.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.271 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:23.372 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33955/271518 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:23 smithi175 ceph-mon[101434]: osdmap e147: 8 total, 8 up, 7 in 2024-08-17T17:23:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33955/271518 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:23 smithi064 ceph-mon[104516]: osdmap e147: 8 total, 8 up, 7 in 2024-08-17T17:23:23.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33955/271518 objects degraded (12.506%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:23:23.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:23 smithi064 ceph-mon[109246]: osdmap e147: 8 total, 8 up, 7 in 2024-08-17T17:23:23.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:23.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.888 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:24.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:24.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:24.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:24.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:24.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:24.240 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:23] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.51.0" 2024-08-17T17:23:24.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:24.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:24.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:24.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:24.545 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:24.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:24 smithi175 ceph-mon[101434]: pgmap v418: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 34071/264840 objects degraded (12.865%); 17048/264840 objects misplaced (6.437%); 21 MiB/s, 332 objects/s recovering 2024-08-17T17:23:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:24 smithi175 ceph-mon[101434]: osdmap e148: 8 total, 8 up, 7 in 2024-08-17T17:23:24.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:24 smithi064 ceph-mon[104516]: pgmap v418: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 34071/264840 objects degraded (12.865%); 17048/264840 objects misplaced (6.437%); 21 MiB/s, 332 objects/s recovering 2024-08-17T17:23:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:24 smithi064 ceph-mon[104516]: osdmap e148: 8 total, 8 up, 7 in 2024-08-17T17:23:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:24 smithi064 ceph-mon[109246]: pgmap v418: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 34071/264840 objects degraded (12.865%); 17048/264840 objects misplaced (6.437%); 21 MiB/s, 332 objects/s recovering 2024-08-17T17:23:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:24 smithi064 ceph-mon[109246]: osdmap e148: 8 total, 8 up, 7 in 2024-08-17T17:23:24.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:24.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:24.909 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:24.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:25.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:25.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:25.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:25.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:25.481 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:25.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:25.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:25.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:25.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:25.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:25.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.997 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:26.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:26.156 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:26.258 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.288 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:26.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:26.426 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:26.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:26.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:26 smithi064 ceph-mon[104516]: pgmap v421: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34071/263328 objects degraded (12.939%); 17048/263328 objects misplaced (6.474%); 13 MiB/s, 214 objects/s recovering 2024-08-17T17:23:26.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:26 smithi064 ceph-mon[109246]: pgmap v421: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34071/263328 objects degraded (12.939%); 17048/263328 objects misplaced (6.474%); 13 MiB/s, 214 objects/s recovering 2024-08-17T17:23:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:26 smithi175 ceph-mon[101434]: pgmap v421: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34071/263328 objects degraded (12.939%); 17048/263328 objects misplaced (6.474%); 13 MiB/s, 214 objects/s recovering 2024-08-17T17:23:26.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:26.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:26.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:26.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:26.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:26.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:26.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:27.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:27.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:27.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:27.261 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:27.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:27.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:27.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.551 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:27.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:27.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:27.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:27.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:27.837 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:27.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:27.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:27.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:28.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:28.129 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:28.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:28.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:28.257 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:28.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:28.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:28.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:28.535 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:28.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:28 smithi064 ceph-mon[104516]: pgmap v422: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 34135/260355 objects degraded (13.111%); 16224/260355 objects misplaced (6.231%); 14 MiB/s, 223 objects/s recovering 2024-08-17T17:23:28.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34135/260355 objects degraded (13.111%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:28.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:28 smithi064 ceph-mon[109246]: pgmap v422: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 34135/260355 objects degraded (13.111%); 16224/260355 objects misplaced (6.231%); 14 MiB/s, 223 objects/s recovering 2024-08-17T17:23:28.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34135/260355 objects degraded (13.111%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:28.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:28 smithi175 ceph-mon[101434]: pgmap v422: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 34135/260355 objects degraded (13.111%); 16224/260355 objects misplaced (6.231%); 14 MiB/s, 223 objects/s recovering 2024-08-17T17:23:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34135/260355 objects degraded (13.111%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:28.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:28.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.815 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:28.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:28.979 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:28.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:29.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:29.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:29.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:29.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:29.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:29.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:29.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:29.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:29.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:29.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:29.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:29.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:29.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:29.992 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.015 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:23:30.015 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180681 5.d e148/135 5:b68a2b37:::benchmark_data_smithi064_147470_object48737:head v 148'11318, mlcod=114'1156)", 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.954227+0000", 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.057289743999999997, 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.057318369000000001, 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954227+0000", 2024-08-17T17:23:30.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954227+0000", 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954228+0000", 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2e-07 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954232+0000", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.476e-06 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954243+0000", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1147e-05 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.954249+0000", 2024-08-17T17:23:30.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0090000000000004e-06 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.973246+0000", 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018996348999999999 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.973270+0000", 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4774999999999999e-05 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.019 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180692 5.d e148/135 5:be1ac707:::benchmark_data_smithi064_147470_object48748:head v 148'11319, mlcod=114'1156)", 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.964459+0000", 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.047057514000000002, 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047139729999999998, 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964459+0000", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964459+0000", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964460+0000", 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0499999999999999e-07 2024-08-17T17:23:30.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964463+0000", 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6009999999999999e-06 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964470+0000", 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7569999999999998e-06 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.964475+0000", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6469999999999997e-06 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.986021+0000", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021546394999999999 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.986047+0000", 2024-08-17T17:23:30.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6010999999999999e-05 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:180708 5.5 5:a63f596a:::benchmark_data_smithi064_147470_object48764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.967483+0000", 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.044033910000000002, 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044165155999999997, 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.024 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 180708 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967483+0000", 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967483+0000", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967485+0000", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7859999999999999e-06 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967491+0000", 2024-08-17T17:23:30.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8139999999999997e-06 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967503+0000", 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2782000000000001e-05 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.967512+0000", 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4829999999999999e-06 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.008010+0000", 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040497963999999997 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.008146+0000", 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136141 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.008281+0000", 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013489099999999999 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180713 5.d e148/135 5:b38b8987:::benchmark_data_smithi064_147470_object48769:head v 148'11320, mlcod=114'1156)", 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.985195+0000", 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026321462, 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026513096999999999, 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.029 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985195+0000", 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985195+0000", 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985197+0000", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217e-06 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985206+0000", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9679999999999995e-06 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985219+0000", 2024-08-17T17:23:30.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3224000000000001e-05 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985228+0000", 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9709999999999993e-06 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.008350+0000", 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023122417999999999 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.008372+0000", 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2163000000000001e-05 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180712 5.b e148/135 5:dd08adca:::benchmark_data_smithi064_147470_object48768:head v 148'11226, mlcod=114'1173)", 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.985241+0000", 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.026275662000000002, 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026519133, 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:30.033 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985241+0000", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985241+0000", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985242+0000", 2024-08-17T17:23:30.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.03e-07 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985246+0000", 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0539999999999996e-06 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985252+0000", 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3149999999999997e-06 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.985257+0000", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0119999999999996e-06 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.993931+0000", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086742599999999996 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.993962+0000", 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0443e-05 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011699+0000", 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017737523000000002 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180714 5.8 e148/111 5:1bcedd04:::benchmark_data_smithi064_147470_object48770:head v 148'11378, mlcod=148'11375)", 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.988424+0000", 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.023092774999999999, 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023425700000000001, 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.037 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988424+0000", 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988424+0000", 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988425+0000", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.333e-06 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988435+0000", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1649999999999995e-06 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988455+0000", 2024-08-17T17:23:30.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0834000000000001e-05 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988522+0000", 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7074000000000002e-05 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.989972+0000", 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014497830000000001 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.989999+0000", 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6339999999999999e-05 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:180718 5.5 5.f9692745 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e148)", 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.988606+0000", 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.022910525000000001, 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023303013000000001, 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.041 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 180718 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988606+0000", 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988606+0000", 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988608+0000", 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.629e-06 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988613+0000", 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9649999999999999e-06 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988629+0000", 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5962000000000001e-05 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.988637+0000", 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8939999999999993e-06 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180717 5.8 e148/111 5:16dd1438:::benchmark_data_smithi064_147470_object48773:head v 148'11379, mlcod=148'11375)", 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.994921+0000", 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016595622000000001, 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01703677, 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994921+0000", 2024-08-17T17:23:30.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994921+0000", 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994922+0000", 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0759999999999999e-06 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994930+0000", 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7130000000000005e-06 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994942+0000", 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2007999999999999e-05 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.994951+0000", 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5280000000000003e-06 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.007363+0000", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012412233 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.007384+0000", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1058999999999999e-05 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180720 5.7 e148/135 5:e8e413b6:::benchmark_data_smithi064_147470_object48776:head v 148'11367, mlcod=114'1161)", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:30.001575+0000", 2024-08-17T17:23:30.049 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0099414710000000003, 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010437299000000001, 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001575+0000", 2024-08-17T17:23:30.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001575+0000", 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001577+0000", 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4929999999999999e-06 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001587+0000", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0147e-05 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001609+0000", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1835999999999999e-05 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001619+0000", 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0074999999999999e-05 2024-08-17T17:23:30.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.002383+0000", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076421499999999995 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.002412+0000", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9238e-05 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.010183+0000", 2024-08-17T17:23:30.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077708680000000002 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180722 5.c e148/128 5:35ad17af:::benchmark_data_smithi064_147470_object48778:head v 148'11398, mlcod=148'11396)", 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:30.001644+0000", 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0098729979999999991, 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010437694000000001, 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:30.054 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001644+0000", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001644+0000", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001644+0000", 2024-08-17T17:23:30.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0699999999999998e-07 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001648+0000", 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.658e-06 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001654+0000", 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9340000000000003e-06 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001658+0000", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3989999999999997e-06 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001761+0000", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010285000000000001 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.001782+0000", 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1466000000000001e-05 2024-08-17T17:23:30.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.010234+0000", 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084516739999999993 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180721 5.8 e148/111)", 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:30.006936+0000", 2024-08-17T17:23:30.058 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.004581239, 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051972479999999998, 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006936+0000", 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006936+0000", 2024-08-17T17:23:30.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006937+0000", 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.153e-06 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006944+0000", 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006963+0000", 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8737e-05 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.006972+0000", 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0950000000000002e-06 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.061 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180723 5.7 e148/135)", 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:30.011908+0000", 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4294967295.9996095, 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026310400000000001, 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011908+0000", 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011908+0000", 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011908+0000", 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2799999999999995e-07 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011913+0000", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0459999999999996e-06 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011927+0000", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4153e-05 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:30.011935+0000", 2024-08-17T17:23:30.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4420000000000001e-06 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:180672 5.d e148/135 5:b14fc890:::benchmark_data_smithi064_147470_object48728:head v 148'11317, mlcod=114'1156)", 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:29.938364+0000", 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.073152532000000006, 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073848430000000007, 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:30.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938364+0000", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938364+0000", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938365+0000", 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210000000000001e-06 2024-08-17T17:23:30.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938372+0000", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.131e-06 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938385+0000", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2889e-05 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.938394+0000", 2024-08-17T17:23:30.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6649999999999992e-06 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.945494+0000", 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070999330000000001 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:29.945521+0000", 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7226e-05 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.068 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:30.069 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:23:30.069 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 13 2024-08-17T17:23:30.069 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:23:30.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:30.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:30.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:30.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.264 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:23:30.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:30.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[104516]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 34135/255033 objects degraded (13.385%); 16224/255033 objects misplaced (6.362%); 13 MiB/s, 212 objects/s recovering 2024-08-17T17:23:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4161400564' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[109246]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 34135/255033 objects degraded (13.385%); 16224/255033 objects misplaced (6.362%); 13 MiB/s, 212 objects/s recovering 2024-08-17T17:23:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4161400564' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:30.504 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:30.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:30.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:30.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:30 smithi175 ceph-mon[101434]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 34135/255033 objects degraded (13.385%); 16224/255033 objects misplaced (6.362%); 13 MiB/s, 212 objects/s recovering 2024-08-17T17:23:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4161400564' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:30.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:30.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:30.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:30.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:31.039 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.064 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:31.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:31.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:23:31.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:31.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:31.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:31.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:31.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:31.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:31.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[104516]: osdmap e149: 8 total, 8 up, 7 in 2024-08-17T17:23:31.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:31.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:31.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:31 smithi064 ceph-mon[109246]: osdmap e149: 8 total, 8 up, 7 in 2024-08-17T17:23:31.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.524 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:31.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:31.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:31 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:31.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:31 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:31.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:31 smithi175 ceph-mon[101434]: osdmap e149: 8 total, 8 up, 7 in 2024-08-17T17:23:31.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:31.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:31.816 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:31.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:31.918 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:32.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:32.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:32.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:32.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.201 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:32.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.304 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:23:32.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:32.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:32.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:32.500 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:32.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:32.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:32.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:32 smithi175 ceph-mon[101434]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 34135/254439 objects degraded (13.416%); 16026/254439 objects misplaced (6.299%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:23:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:23:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:32 smithi064 ceph-mon[104516]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 34135/254439 objects degraded (13.416%); 16026/254439 objects misplaced (6.299%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:23:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:23:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:32 smithi064 ceph-mon[109246]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 34135/254439 objects degraded (13.416%); 16026/254439 objects misplaced (6.299%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:23:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:23:32.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:32.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:33.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.070 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:33.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:33.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:33.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:33.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:33.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:33.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:33.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:33.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:33.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:33.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[104516]: osdmap e150: 8 total, 8 up, 7 in 2024-08-17T17:23:33.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34135/254439 objects degraded (13.416%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:33.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:33.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[109246]: osdmap e150: 8 total, 8 up, 7 in 2024-08-17T17:23:33.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34135/254439 objects degraded (13.416%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:33.637 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:33.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:33.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/721872399' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:33 smithi175 ceph-mon[101434]: osdmap e150: 8 total, 8 up, 7 in 2024-08-17T17:23:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34135/254439 objects degraded (13.416%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:33.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:23:33.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:33.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:33.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:33.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:33.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:33.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:23:34.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:34.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:34.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:34.245 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:33] "GET /metrics HTTP/1.1" 200 36674 "" "Prometheus/2.51.0" 2024-08-17T17:23:34.293 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:34.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.420 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:34.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:34.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:34.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:34 smithi064 ceph-mon[104516]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 34230/248778 objects degraded (13.759%); 15652/248778 objects misplaced (6.292%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:23:34.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:34 smithi064 ceph-mon[109246]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 34230/248778 objects degraded (13.759%); 15652/248778 objects misplaced (6.292%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:23:34.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:34.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:34 smithi175 ceph-mon[101434]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 34230/248778 objects degraded (13.759%); 15652/248778 objects misplaced (6.292%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:23:34.669 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:34.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:34.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:34.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:34.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:34.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:34.985 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:35.087 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:35.108 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:35.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:35.218 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:35.269 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:35.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.401 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:35.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:35.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/452920912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/452920912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.541 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:35.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:35.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:35.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/452920912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:35.792 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:35.816 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:35.840 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:23:35.840 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:23:35.840 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:23:35.840 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:23:35.840 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:23:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:35.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:35.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:35.918 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:35.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:36.049 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:36.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:36.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:36.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:36.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:36.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:36.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:36 smithi064 ceph-mon[104516]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 34230/246132 objects degraded (13.907%); 15652/246132 objects misplaced (6.359%); 720 KiB/s, 11 objects/s recovering 2024-08-17T17:23:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:36 smithi064 ceph-mon[109246]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 34230/246132 objects degraded (13.907%); 15652/246132 objects misplaced (6.359%); 720 KiB/s, 11 objects/s recovering 2024-08-17T17:23:36.473 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:36.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:36.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:36.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:36 smithi175 ceph-mon[101434]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 34230/246132 objects degraded (13.907%); 15652/246132 objects misplaced (6.359%); 720 KiB/s, 11 objects/s recovering 2024-08-17T17:23:36.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:36.778 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:36.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:36.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:36.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:37.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:37.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:37.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:37.327 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:37.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:37.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:37.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:37.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:37.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:37.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:37.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:37.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:37.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.886 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:38.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:38.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:38.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:38.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:38.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:38.426 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:38.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:38.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:38 smithi175 ceph-mon[101434]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 34331/242595 objects degraded (14.152%); 15071/242595 objects misplaced (6.212%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34331/242595 objects degraded (14.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:38.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:38.687 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:38 smithi064 ceph-mon[104516]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 34331/242595 objects degraded (14.152%); 15071/242595 objects misplaced (6.212%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34331/242595 objects degraded (14.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:38 smithi064 ceph-mon[109246]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 34331/242595 objects degraded (14.152%); 15071/242595 objects misplaced (6.212%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34331/242595 objects degraded (14.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:38.789 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:38.818 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:38.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:38.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:38.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:39.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.101 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:39.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:39.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:39.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:39.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:39.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:39.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:39.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:39.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:39.649 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:39.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:39.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:39.877 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:39.877 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:39.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:39.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:40.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:40.054 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:40.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:40.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:40.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:40.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:40.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:40.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:40.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:40.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:40.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:40 smithi175 ceph-mon[101434]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 34331/236502 objects degraded (14.516%); 15071/236502 objects misplaced (6.372%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:23:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:40.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:40 smithi064 ceph-mon[104516]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 34331/236502 objects degraded (14.516%); 15071/236502 objects misplaced (6.372%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:23:40.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:40.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:40 smithi064 ceph-mon[109246]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 34331/236502 objects degraded (14.516%); 15071/236502 objects misplaced (6.372%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:23:40.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:40.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:40.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:40.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:40.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:40.928 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:41.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:41.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:41.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:23:41.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.381 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:23:41.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3495080501' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:41 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3495080501' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:41 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.529 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:41.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.630 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3495080501' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:41 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:41.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.664 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:41.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:41.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:41.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:41.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:41.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:42.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:42.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:42.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:42.187 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:42.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:42.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:42.386 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:23:42.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:42.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:42.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:23:42.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:42.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:42 smithi175 ceph-mon[101434]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 34331/235887 objects degraded (14.554%); 14867/235887 objects misplaced (6.303%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:42 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:42 smithi175 ceph-mon[101434]: osdmap e151: 8 total, 8 up, 7 in 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:42.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[104516]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 34331/235887 objects degraded (14.554%); 14867/235887 objects misplaced (6.303%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[104516]: osdmap e151: 8 total, 8 up, 7 in 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[109246]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 34331/235887 objects degraded (14.554%); 14867/235887 objects misplaced (6.303%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:42.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[109246]: osdmap e151: 8 total, 8 up, 7 in 2024-08-17T17:23:42.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:42.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:42.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:42.874 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:42.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:43.018 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:43.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:43.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:43.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:43.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:43.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:43.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:43.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:43.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:43.646 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:43.650 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:43 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:43 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:43 smithi175 ceph-mon[101434]: osdmap e152: 8 total, 8 up, 7 in 2024-08-17T17:23:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34331/235887 objects degraded (14.554%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:43.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[104516]: osdmap e152: 8 total, 8 up, 7 in 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34331/235887 objects degraded (14.554%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1158146929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[109246]: osdmap e152: 8 total, 8 up, 7 in 2024-08-17T17:23:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34331/235887 objects degraded (14.554%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:43.701 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:43.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:43.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:44.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:44.078 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:43] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-08-17T17:23:44.090 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:44.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:44.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:44.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:44.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:44.461 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:44.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:44.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:44 smithi175 ceph-mon[101434]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 34426/229752 objects degraded (14.984%); 14455/229752 objects misplaced (6.292%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:44 smithi064 ceph-mon[104516]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 34426/229752 objects degraded (14.984%); 14455/229752 objects misplaced (6.292%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:44.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:44 smithi064 ceph-mon[109246]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 34426/229752 objects degraded (14.984%); 14455/229752 objects misplaced (6.292%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:23:44.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:44.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:44.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:44.997 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:45.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:45.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:45.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:45.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:45.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:45.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:45.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.547 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:45.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:45.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:23:45.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.725 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:45.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:45.854 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:45.868 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:45.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:46.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:46.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:46.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:46.239 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:46.271 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:46.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:46.408 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:46.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:46.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:46.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:46.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:46 smithi175 ceph-mon[101434]: pgmap v435: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 34426/226785 objects degraded (15.180%); 14455/226785 objects misplaced (6.374%); 767 KiB/s, 11 objects/s recovering 2024-08-17T17:23:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:46 smithi064 ceph-mon[104516]: pgmap v435: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 34426/226785 objects degraded (15.180%); 14455/226785 objects misplaced (6.374%); 767 KiB/s, 11 objects/s recovering 2024-08-17T17:23:46.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:46 smithi064 ceph-mon[109246]: pgmap v435: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 34426/226785 objects degraded (15.180%); 14455/226785 objects misplaced (6.374%); 767 KiB/s, 11 objects/s recovering 2024-08-17T17:23:46.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:46.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:46.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:46.926 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:46.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:47.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:47.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:47.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:47.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:47.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:47.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:47.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:47.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:47.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:23:47.531 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:47.565 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:23:47.565 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:47.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:47.607 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:23:47.607 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:47.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:47.640 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:23:47.640 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:47.675 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:23:47.675 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:47.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:23:47.706 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:47.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:47.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:23:47.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:47.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:47.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:48.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:48.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:48.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:48.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:48.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.325 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:48.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:48.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:48.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:48.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:48.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:48 smithi175 ceph-mon[101434]: pgmap v436: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 34520/223170 objects degraded (15.468%); 13830/223170 objects misplaced (6.197%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1207750193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34520/223170 objects degraded (15.468%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:48.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:48.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[104516]: pgmap v436: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 34520/223170 objects degraded (15.468%); 13830/223170 objects misplaced (6.197%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:48.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1207750193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:48.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34520/223170 objects degraded (15.468%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:48.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[109246]: pgmap v436: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 34520/223170 objects degraded (15.468%); 13830/223170 objects misplaced (6.197%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:48.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1207750193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:48.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34520/223170 objects degraded (15.468%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:48.729 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:48.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:48.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:48.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:48.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140398 5.5 5:a79e626c:::benchmark_data_smithi064_147470_object8454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:19.514578+0000", 2024-08-17T17:23:49.016 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 89.494342109000002, 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14094716600000001, 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140398 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514578+0000", 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514578+0000", 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514580+0000", 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5689999999999999e-06 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514585+0000", 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4389999999999999e-06 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514601+0000", 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5906999999999999e-05 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.514609+0000", 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7989999999999994e-06 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.541813+0000", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027203914999999999 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.541958+0000", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144451 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.611601+0000", 2024-08-17T17:23:49.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.069643514000000004 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.611695+0000", 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4106999999999999e-05 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.611989+0000", 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029427300000000001 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.637664+0000", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025674947 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.637731+0000", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6647000000000001e-05 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.637750+0000", 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9006000000000001e-05 2024-08-17T17:23:49.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.637807+0000", 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6464e-05 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:19.655526+0000", 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017719128000000001 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140775 5.5 5:abc89a9a:::benchmark_data_smithi064_147470_object8831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 88.836568421999999, 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1096915650000001, 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140775 2024-08-17T17:23:49.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172354+0000", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.843e-06 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172361+0000", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1069999999999999e-06 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172374+0000", 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3142999999999999e-05 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172382+0000", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1859999999999992e-06 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188729+0000", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016346823 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188871+0000", 2024-08-17T17:23:49.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141722 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239525+0000", 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050654413000000002 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239649+0000", 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012354 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:23:49.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239798+0000", 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014950799999999999 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.252143+0000", 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012344554000000001 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.252150+0000", 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9240000000000001e-06 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.272388+0000", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.020237721 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.272596+0000", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020802900000000001 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281984+0000", 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093880230000000005 2024-08-17T17:23:49.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282044+0000", 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0028999999999997e-05 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140784 5.5 5:af152c95:::benchmark_data_smithi064_147470_object8840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 88.820997453999993, 2024-08-17T17:23:49.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.093724857, 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140784 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:23:49.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187925+0000", 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0899999999999999e-06 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187929+0000", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0339999999999998e-06 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187940+0000", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.082e-05 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187946+0000", 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3300000000000004e-06 2024-08-17T17:23:49.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188969+0000", 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010226149999999999 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.189359+0000", 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038968800000000001 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:23:49.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.189775+0000", 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041620399999999999 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.250918+0000", 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.061142672000000002 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.250931+0000", 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3404000000000001e-05 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271603+0000", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0206717270000001 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271704+0000", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010087200000000001 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281568+0000", 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098639290000000004 2024-08-17T17:23:49.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281648+0000", 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0471999999999998e-05 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163089 5.d e142/135 5:b5fe6791:::benchmark_data_smithi064_147470_object31145:head v 142'10240, mlcod=114'1156)", 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.599684+0000", 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.409236505999999, 2024-08-17T17:23:49.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18089089699999999, 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599684+0000", 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599684+0000", 2024-08-17T17:23:49.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599685+0000", 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2200000000000004e-07 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599690+0000", 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5260000000000001e-06 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599701+0000", 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1141999999999999e-05 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.599707+0000", 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.541e-06 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.623023+0000", 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02331571 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.623045+0000", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1994e-05 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780543+0000", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.157498789 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780575+0000", 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1673e-05 2024-08-17T17:23:49.042 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163093 5.d e142/135 5:bb361892:::benchmark_data_smithi064_147470_object31149:head v 142'10241, mlcod=114'1156)", 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.614242+0000", 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.394678204999998, 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.166364548, 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614242+0000", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614242+0000", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614243+0000", 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.023e-06 2024-08-17T17:23:49.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614250+0000", 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3269999999999997e-06 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614264+0000", 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4207e-05 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614335+0000", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1333000000000003e-05 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635352+0000", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021016762000000001 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635375+0000", 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3368000000000001e-05 2024-08-17T17:23:49.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780578+0000", 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.145202422 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780607+0000", 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9105999999999999e-05 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163101 5.d e142/135 5:b7992808:::benchmark_data_smithi064_147470_object31157:head v 142'10242, mlcod=114'1156)", 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.614385+0000", 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.394535669, 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18954248200000001, 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614385+0000", 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614385+0000", 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614385+0000", 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8999999999999998e-07 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614393+0000", 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7530000000000001e-06 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614413+0000", 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0237999999999998e-05 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614419+0000", 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4450000000000004e-06 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.648067+0000", 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033648428000000001 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.648089+0000", 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2333000000000001e-05 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.803877+0000", 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15578777399999999 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.803927+0000", 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0220999999999999e-05 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:163108 5.5 5:a8aebfb6:::benchmark_data_smithi064_147470_object31164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.614584+0000", 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.394337055999998, 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25068533999999998, 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 163108 2024-08-17T17:23:49.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614584+0000", 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614584+0000", 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614585+0000", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5799999999999999e-06 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614593+0000", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8159999999999994e-06 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614605+0000", 2024-08-17T17:23:49.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1717e-05 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.614612+0000", 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5120000000000002e-06 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.803972+0000", 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18935993000000001 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.804100+0000", 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127568 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.804220+0000", 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011990899999999999 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.817269+0000", 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013049088 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.817310+0000", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1449000000000002e-05 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.864946+0000", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047635599000000001 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.864995+0000", 2024-08-17T17:23:49.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9648000000000003e-05 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.865269+0000", 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027352399999999998 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163102 5.d e142/135 5:b3bf3a68:::benchmark_data_smithi064_147470_object31158:head v 142'10243, mlcod=114'1156)", 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.619320+0000", 2024-08-17T17:23:49.059 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.389600895999997, 2024-08-17T17:23:49.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.224710084, 2024-08-17T17:23:49.060 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.060 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.060 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.060 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619320+0000", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619320+0000", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619320+0000", 2024-08-17T17:23:49.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9400000000000005e-07 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619326+0000", 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4489999999999998e-06 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619335+0000", 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3710000000000006e-06 2024-08-17T17:23:49.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.619342+0000", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5649999999999998e-06 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780649+0000", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16130699800000001 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780664+0000", 2024-08-17T17:23:49.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5603999999999998e-05 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.843997+0000", 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063332327999999993 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.844030+0000", 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3074999999999998e-05 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:163115 5.5 5:a4079065:::benchmark_data_smithi064_147470_object31171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.630345+0000", 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.378575777000002, 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25548446200000002, 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 163115 2024-08-17T17:23:49.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630345+0000", 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630345+0000", 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630347+0000", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8810000000000001e-06 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630352+0000", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6629999999999998e-06 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630365+0000", 2024-08-17T17:23:49.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2337000000000001e-05 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.630372+0000", 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5050000000000001e-06 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.857813+0000", 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22744115200000001 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.857967+0000", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015352399999999999 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.858142+0000", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017537100000000001 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875356+0000", 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017213284999999998 2024-08-17T17:23:49.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875400+0000", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3995000000000001e-05 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.885461+0000", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010061274 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.885639+0000", 2024-08-17T17:23:49.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017842800000000001 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.885829+0000", 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019004699999999999 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:163119 5.5 5:a0f54bcb:::benchmark_data_smithi064_147470_object31175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.635969+0000", 2024-08-17T17:23:49.071 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.372951110000002, 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.250341751, 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 163119 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635969+0000", 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635969+0000", 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635971+0000", 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6449999999999999e-06 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635976+0000", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1479999999999997e-06 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635987+0000", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0781000000000001e-05 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.635994+0000", 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9879999999999998e-06 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875431+0000", 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23943721600000001 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875560+0000", 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012907500000000001 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:49.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875719+0000", 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158531 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.885406+0000", 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096870029999999996 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.885991+0000", 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058472400000000001 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.886088+0000", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7518999999999999e-05 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.886135+0000", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6468999999999998e-05 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.886311+0000", 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017665199999999999 2024-08-17T17:23:49.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163121 5.d e142/135 5:b0d99972:::benchmark_data_smithi064_147470_object31177:head v 142'10244, mlcod=114'1156)", 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.640472+0000", 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.368448291, 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20358159300000001, 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.078 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640472+0000", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640472+0000", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640473+0000", 2024-08-17T17:23:49.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8299999999999998e-07 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640477+0000", 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9550000000000002e-06 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640484+0000", 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.148e-06 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.640489+0000", 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7929999999999997e-06 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.792443+0000", 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15195483900000001 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.792464+0000", 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0727e-05 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.844034+0000", 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.051569624000000001 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.844054+0000", 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0123999999999999e-05 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:163123 5.5 5:aff6c9bc:::benchmark_data_smithi064_147470_object31179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.642527+0000", 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.366393442000003, 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25308380600000002, 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 163123 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642527+0000", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642527+0000", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642529+0000", 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.525e-06 2024-08-17T17:23:49.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642533+0000", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3680000000000004e-06 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642545+0000", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1741e-05 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.642553+0000", 2024-08-17T17:23:49.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8809999999999996e-06 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.880299+0000", 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.237746614 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.880431+0000", 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013175799999999999 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.880741+0000", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030998199999999999 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.886073+0000", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053316070000000004 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.886333+0000", 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000260442 2024-08-17T17:23:49.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.895234+0000", 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089006789999999999 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.895453+0000", 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000219302 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.895611+0000", 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015790699999999999 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163122 5.d e142/135 5:b913fcd1:::benchmark_data_smithi064_147470_object31178:head v 142'10245, mlcod=114'1156)", 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.645600+0000", 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.363320625, 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22975962799999999, 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.089 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645600+0000", 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645600+0000", 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645601+0000", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8599999999999998e-07 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645606+0000", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8440000000000003e-06 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645617+0000", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0356e-05 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.645623+0000", 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9510000000000003e-06 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.803988+0000", 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15836551099999999 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.804012+0000", 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4158000000000001e-05 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875315+0000", 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071302518999999995 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875360+0000", 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4703000000000002e-05 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163132 5.d e142/135 5:b563a354:::benchmark_data_smithi064_147470_object31188:head v 142'10246, mlcod=114'1156)", 2024-08-17T17:23:49.093 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.661738+0000", 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.347182779000001, 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21365271699999999, 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661738+0000", 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661738+0000", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661739+0000", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0999999999999997e-07 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661746+0000", 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5639999999999999e-06 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661757+0000", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0927000000000001e-05 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.661764+0000", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7989999999999996e-06 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.817339+0000", 2024-08-17T17:23:49.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15557559200000001 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.817363+0000", 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3760999999999998e-05 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875364+0000", 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058000358000000002 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875391+0000", 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6906e-05 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.34626.0:163136 5.d e142/135 5:b3372ef0:::benchmark_data_smithi064_147470_object31192:head v 142'10247, mlcod=114'1156)", 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.674792+0000", 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.334128749000001, 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20063081499999999, 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.098 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.674792+0000", 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.674792+0000", 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.674793+0000", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4249999999999999e-06 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.674803+0000", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6549999999999999e-06 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780241+0000", 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10543799700000001 2024-08-17T17:23:49.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.780251+0000", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7210000000000006e-06 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.830243+0000", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049991995999999997 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.830265+0000", 2024-08-17T17:23:49.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2229999999999999e-05 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875393+0000", 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045127887999999998 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.875423+0000", 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9903000000000002e-05 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.102 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176891 5.3 5:c4b25f19:::benchmark_data_smithi064_147470_object44947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.896655383999999, 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.021502071, 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.103 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176891 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112267+0000", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.283e-06 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112272+0000", 2024-08-17T17:23:49.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4660000000000001e-06 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112283+0000", 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.083e-05 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112289+0000", 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.015e-06 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112488+0000", 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019912400000000001 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112514+0000", 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5990999999999999e-05 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126288+0000", 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013773979 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126405+0000", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117107 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126520+0000", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114629 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133526+0000", 2024-08-17T17:23:49.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070063629999999998 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133602+0000", 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6087999999999994e-05 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133673+0000", 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0895999999999993e-05 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133724+0000", 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0569999999999999e-05 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133767+0000", 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3730000000000003e-05 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176912 5.3 5:c2336cb6:::benchmark_data_smithi064_147470_object44968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:23:49.111 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.855114897, 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98830326099999999, 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176912 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5699999999999998e-07 2024-08-17T17:23:49.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153808+0000", 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3300000000000001e-06 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153815+0000", 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4289999999999998e-06 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153820+0000", 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.203e-06 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.159421+0000", 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056008500000000001 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.159432+0000", 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1501000000000001e-05 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130238+0000", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97080522899999999 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130361+0000", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012380099999999999 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130454+0000", 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2539000000000003e-05 2024-08-17T17:23:49.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133661+0000", 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032068109999999999 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.137820+0000", 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041596580000000001 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142004+0000", 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041835850000000001 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142062+0000", 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7779999999999999e-05 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142109+0000", 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7088e-05 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.118 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176939 5.3 5:c2db8817:::benchmark_data_smithi064_147470_object44995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.814547001000001, 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95685923799999995, 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.119 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176939 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194375+0000", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.24e-06 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194380+0000", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194399+0000", 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9885999999999999e-05 2024-08-17T17:23:49.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194407+0000", 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3050000000000004e-06 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194613+0000", 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000206255 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194625+0000", 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1481000000000001e-05 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133903+0000", 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93927822900000002 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.134036+0000", 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133688 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.134155+0000", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011839 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142119+0000", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079641769999999994 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142165+0000", 2024-08-17T17:23:49.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5624000000000002e-05 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151114+0000", 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089488929999999994 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151184+0000", 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0588999999999998e-05 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151233+0000", 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8721000000000003e-05 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176959 5.3 5:ca2aafac:::benchmark_data_smithi064_147470_object45015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.234576+0000", 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.774344304, 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91688450700000002, 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.126 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176959 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234576+0000", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234576+0000", 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234578+0000", 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.832e-06 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234585+0000", 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.686e-06 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234603+0000", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7838000000000001e-05 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234612+0000", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7589999999999992e-06 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234933+0000", 2024-08-17T17:23:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032107700000000002 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.234957+0000", 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4031000000000001e-05 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.137851+0000", 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.902893214 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.137975+0000", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012392499999999999 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.138094+0000", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011930300000000001 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142124+0000", 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040296799999999999 2024-08-17T17:23:49.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151283+0000", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091589549999999999 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151333+0000", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0167999999999998e-05 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151421+0000", 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7996000000000002e-05 2024-08-17T17:23:49.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151461+0000", 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0043000000000003e-05 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176992 5.3 5:cc165657:::benchmark_data_smithi064_147470_object45048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.324202+0000", 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.684718152999999, 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.82747317799999998, 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:23:49.133 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176992 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324202+0000", 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324202+0000", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324203+0000", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0100000000000004e-07 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324208+0000", 2024-08-17T17:23:49.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3640000000000001e-06 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324217+0000", 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6029999999999994e-06 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324222+0000", 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9660000000000004e-06 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324262+0000", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9833999999999998e-05 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.324267+0000", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2669999999999997e-06 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142195+0000", 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81792810999999999 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142305+0000", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109789 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142414+0000", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109299 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151294+0000", 2024-08-17T17:23:49.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008879095 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151509+0000", 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000215918 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151552+0000", 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2954999999999998e-05 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:23:49.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151636+0000", 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3338999999999996e-05 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151676+0000", 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9838e-05 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:23:49.140 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:23:49.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:49.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:49.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:49.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:49.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:49.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:49.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:49.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:49.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:49.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:49.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:49.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:49.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:49.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:49.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:49.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:50.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:50.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:50.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:50.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:50.382 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:50.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:50.511 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:50.542 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:50.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:50.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:50.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:50 smithi175 ceph-mon[101434]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 34520/216969 objects degraded (15.910%); 13830/216969 objects misplaced (6.374%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:50.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:50 smithi064 ceph-mon[104516]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 34520/216969 objects degraded (15.910%); 13830/216969 objects misplaced (6.374%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:50.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:50 smithi064 ceph-mon[109246]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 34520/216969 objects degraded (15.910%); 13830/216969 objects misplaced (6.374%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:23:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:50.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:50.789 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:50.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:50.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:50.921 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:50.956 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:51.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:51.114 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:51.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:23:51.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.236 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:51.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:51.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.531 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:51.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:51.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:51.679 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:51.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:51.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:51.748 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:51.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:51.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.914 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:51.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:52.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:52.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:52.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:52.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:52.212 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:52.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:52.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:52.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:52 smithi064 ceph-mon[104516]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 34520/216408 objects degraded (15.951%); 13644/216408 objects misplaced (6.305%); 657 KiB/s, 10 objects/s recovering 2024-08-17T17:23:52.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:52 smithi064 ceph-mon[109246]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 34520/216408 objects degraded (15.951%); 13644/216408 objects misplaced (6.305%); 657 KiB/s, 10 objects/s recovering 2024-08-17T17:23:52.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:52.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:52.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:52.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:52.762 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:52.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:52.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:52.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:52 smithi175 ceph-mon[101434]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 34520/216408 objects degraded (15.951%); 13644/216408 objects misplaced (6.305%); 657 KiB/s, 10 objects/s recovering 2024-08-17T17:23:52.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:53.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:53.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:53.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.360 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:53.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:53.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:53.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.674 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:53.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34520/216408 objects degraded (15.951%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:53.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34520/216408 objects degraded (15.951%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:53.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:53.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34520/216408 objects degraded (15.951%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:53.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:53.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.045 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:54.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:54.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:54.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:54.224 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:23:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:23:53] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-08-17T17:23:54.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:54.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:54.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:54.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:54.471 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:54.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:54 smithi064 ceph-mon[104516]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 34614/210450 objects degraded (16.448%); 13247/210450 objects misplaced (6.295%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:54.487 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:54 smithi064 ceph-mon[109246]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 34614/210450 objects degraded (16.448%); 13247/210450 objects misplaced (6.295%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:54.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:54.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:54.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.783 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:54 smithi175 ceph-mon[101434]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 34614/210450 objects degraded (16.448%); 13247/210450 objects misplaced (6.295%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:54.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:54.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:54.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:55.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:55.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:55.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:55.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:55.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:55.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:55.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:55.419 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:55.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:55.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:55.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:55.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:55.765 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:55.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:55.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:56.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:56.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:56.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 34614/207273 objects degraded (16.700%); 13247/207273 objects misplaced (6.391%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 34614/207273 objects degraded (16.700%); 13247/207273 objects misplaced (6.391%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:56.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:56.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 34614/207273 objects degraded (16.700%); 13247/207273 objects misplaced (6.391%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:23:56.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:56.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:56.448 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:23:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:56.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:56.725 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:56.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:56.852 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:56.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:56.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:57.136 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:57.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:57.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:57.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:57.410 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:57.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:57.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:57.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:57.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[104516]: osdmap e153: 8 total, 8 up, 7 in 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2765184317' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[109246]: osdmap e153: 8 total, 8 up, 7 in 2024-08-17T17:23:57.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2765184317' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:57 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:57.858 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:57.880 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:23:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3120078696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:23:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:57 smithi175 ceph-mon[101434]: osdmap e153: 8 total, 8 up, 7 in 2024-08-17T17:23:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2765184317' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:57 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:57.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:57.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:58.122 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:58.224 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:58.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:58.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:58.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:58.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:58.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:58.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:58.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.649 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:58.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[104516]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 34705/203778 objects degraded (17.031%); 12702/203778 objects misplaced (6.233%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[104516]: osdmap e154: 8 total, 8 up, 7 in 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34705/203778 objects degraded (17.031%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[109246]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 34705/203778 objects degraded (17.031%); 12702/203778 objects misplaced (6.233%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[109246]: osdmap e154: 8 total, 8 up, 7 in 2024-08-17T17:23:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:23:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34705/203778 objects degraded (17.031%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:58.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:58.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:58.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:58.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:58 smithi175 ceph-mon[101434]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 34705/203778 objects degraded (17.031%); 12702/203778 objects misplaced (6.233%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:23:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:58 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:58 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:58 smithi175 ceph-mon[101434]: osdmap e154: 8 total, 8 up, 7 in 2024-08-17T17:23:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:23:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34705/203778 objects degraded (17.031%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:23:59.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:59.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.178 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:59.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:59.279 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:59.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:59.448 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:59.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:59.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:59.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:59.685 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:23:59.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:59.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:59.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:59.827 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:23:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:59.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:23:59.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:00.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:00.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:00.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:00.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:00.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:00.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:00.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:00.475 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:24:00.475 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:00.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:00.509 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:24:00.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:00.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:00.543 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:24:00.543 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:00.579 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:24:00.579 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:00.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:24:00.617 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:00.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:00.659 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:24:00.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:00.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:00 smithi064 ceph-mon[104516]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 34705/197958 objects degraded (17.531%); 12702/197958 objects misplaced (6.417%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:24:00.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3436751906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:00.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:00 smithi064 ceph-mon[109246]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 34705/197958 objects degraded (17.531%); 12702/197958 objects misplaced (6.417%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:24:00.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3436751906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:00.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:00.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:24:00.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:00.748 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:24:00.748 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:00.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:00.851 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:00 smithi175 ceph-mon[101434]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 34705/197958 objects degraded (17.531%); 12702/197958 objects misplaced (6.417%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:24:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3436751906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:00.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:00.978 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:01.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:24:01.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:01.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:01.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:01.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:01.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:01.400 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:01.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:01.552 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:01.654 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:01.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:01.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:01.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:01.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:01.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:01.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:02.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:02.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:02.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:02.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.235 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:02.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:02.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:02.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:02.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:02.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:02.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:02.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:02.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:02.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:02 smithi064 ceph-mon[104516]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 34705/197355 objects degraded (17.585%); 12502/197355 objects misplaced (6.335%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:24:02.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:02 smithi064 ceph-mon[109246]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 34705/197355 objects degraded (17.585%); 12502/197355 objects misplaced (6.335%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:24:02.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:02.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:02 smithi175 ceph-mon[101434]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 34705/197355 objects degraded (17.585%); 12502/197355 objects misplaced (6.335%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:24:03.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:03.038 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:03.140 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:03.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:03.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.318 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:03.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:03.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:03.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:03.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:03.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:03.573 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:03.613 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:03.614 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:03.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:03.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34705/197355 objects degraded (17.585%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:03.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34705/197355 objects degraded (17.585%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:03.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:03.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:03.852 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:03.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34705/197355 objects degraded (17.585%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:03.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:03.995 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:04.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:04.136 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:04.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:04.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:04.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:04.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:04.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:03] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-08-17T17:24:04.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:04.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:04.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:24:04.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:04.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:04.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:04.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:04 smithi064 ceph-mon[104516]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 34798/191703 objects degraded (18.152%); 12131/191703 objects misplaced (6.328%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:04.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:04 smithi064 ceph-mon[109246]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 34798/191703 objects degraded (18.152%); 12131/191703 objects misplaced (6.328%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:04.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.905 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:04.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:04 smithi175 ceph-mon[101434]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 34798/191703 objects degraded (18.152%); 12131/191703 objects misplaced (6.328%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:05.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:05.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:05.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:05.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:05.277 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:05.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:05.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:05.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:05.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:05.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:05.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:05.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:05.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:06.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:06.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:06.132 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:06.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:06.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:06.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:06.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:06.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:06.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:06.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:06.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:06 smithi064 ceph-mon[104516]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 34798/188925 objects degraded (18.419%); 12131/188925 objects misplaced (6.421%); 695 KiB/s, 10 objects/s recovering 2024-08-17T17:24:06.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:06 smithi064 ceph-mon[109246]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 34798/188925 objects degraded (18.419%); 12131/188925 objects misplaced (6.421%); 695 KiB/s, 10 objects/s recovering 2024-08-17T17:24:06.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:06.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:06.802 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:06.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:06 smithi175 ceph-mon[101434]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 34798/188925 objects degraded (18.419%); 12131/188925 objects misplaced (6.421%); 695 KiB/s, 10 objects/s recovering 2024-08-17T17:24:06.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:06.964 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:06.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:07.027 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:24:07.028 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:24:07.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:07.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:07.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:07.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:07.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:07.351 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:07.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:07.405 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:07.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:07.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:07.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:07.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:07.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:07.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:07.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:08.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:08.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:08.172 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":448,"stamp":"2024-08-17T17:24:06.827831+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4051834877,"num_objects":61782,"num_object_clones":0,"num_object_copies":185346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34886,"num_objects_misplaced":11527,"num_objects_unfound":0,"num_objects_dirty":61782,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":203253,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17229,"num_bytes_recovered":964828324,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132342,"ondisk_log_size":132342,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11135200,"kb_used_data":10200296,"kb_used_omap":33,"kb_used_meta":934750,"kb_avail":645080864,"statfs":{"total":671965249536,"available":660562804736,"internally_reserved":0,"allocated":10445103104,"data_stored":10439039530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":957184587},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-402653184,"num_objects":-6144,"num_object_clones":0,"num_object_copies":-18432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":181,"num_objects_misplaced":-1175,"num_objects_unfound":0,"num_objects_dirty":-6144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6144,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":194,"num_bytes_recovered":12713984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005071"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":162,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.464080+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:23:57.464080+0000","last_peered":"2024-08-17T17:23:57.464080+0000","last_clean":"2024-08-17T17:23:57.464080+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:23:57.464080+0000","last_undegraded":"2024-08-17T17:23:57.464080+0000","last_fullsized":"2024-08-17T17:23:57.464080+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":147,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.479205+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:23:56.479205+0000","last_peered":"2024-08-17T17:23:56.479205+0000","last_clean":"2024-08-17T17:23:56.479205+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:23:56.479205+0000","last_undegraded":"2024-08-17T17:23:56.479205+0000","last_fullsized":"2024-08-17T17:23:56.479205+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478691+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:23:56.478691+0000","last_peered":"2024-08-17T17:23:56.478691+0000","last_clean":"2024-08-17T17:23:56.478691+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:23:56.478691+0000","last_undegraded":"2024-08-17T17:23:56.478691+0000","last_fullsized":"2024-08-17T17:23:56.478691+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":153,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488233+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:23:57.488233+0000","last_peered":"2024-08-17T17:23:57.488233+0000","last_clean":"2024-08-17T17:23:57.488233+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:23:57.488233+0000","last_undegraded":"2024-08-17T17:23:57.488233+0000","last_fullsized":"2024-08-17T17:23:57.488233+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":153,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488954+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:23:57.488954+0000","last_peered":"2024-08-17T17:23:57.488954+0000","last_clean":"2024-08-17T17:23:57.488954+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:23:57.488954+0000","last_undegraded":"2024-08-17T17:23:57.488954+0000","last_fullsized":"2024-08-17T17:23:57.488954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478493+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:23:56.478493+0000","last_peered":"2024-08-17T17:23:56.478493+0000","last_clean":"2024-08-17T17:23:56.478493+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:23:56.478493+0000","last_undegraded":"2024-08-17T17:23:56.478493+0000","last_fullsized":"2024-08-17T17:23:56.478493+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":162,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.465145+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:23:57.465145+0000","last_peered":"2024-08-17T17:23:57.465145+0000","last_clean":"2024-08-17T17:23:57.465145+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:23:57.465145+0000","last_undegraded":"2024-08-17T17:23:57.465145+0000","last_fullsized":"2024-08-17T17:23:57.465145+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.473244+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:23:56.473244+0000","last_peered":"2024-08-17T17:23:56.473244+0000","last_clean":"2024-08-17T17:23:56.473244+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:23:56.473244+0000","last_undegraded":"2024-08-17T17:23:56.473244+0000","last_fullsized":"2024-08-17T17:23:56.473244+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"154'12684","reported_seq":13854,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:05.807740+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:24:05.807740+0000","last_peered":"2024-08-17T17:24:05.807740+0000","last_clean":"2024-08-17T17:24:05.807740+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:24:05.807740+0000","last_undegraded":"2024-08-17T17:24:05.807740+0000","last_fullsized":"2024-08-17T17:24:05.807740+0000","mapping_epoch":135,"log_start":"116'4500","ondisk_log_start":"116'4500","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12684,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":143,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478972+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:23:56.478972+0000","last_peered":"2024-08-17T17:23:56.478972+0000","last_clean":"2024-08-17T17:23:56.478972+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:23:56.478972+0000","last_undegraded":"2024-08-17T17:23:56.478972+0000","last_fullsized":"2024-08-17T17:23:56.478972+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":31,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478384+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:23:56.478384+0000","last_peered":"2024-08-17T17:23:56.478384+0000","last_clean":"2024-08-17T17:23:56.478384+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:23:56.478384+0000","last_undegraded":"2024-08-17T17:23:56.478384+0000","last_fullsized":"2024-08-17T17:23:56.478384+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"154'12663","reported_seq":12761,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.349833+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:24:02.349833+0000","last_peered":"2024-08-17T17:24:02.349833+0000","last_clean":"2024-08-17T17:24:02.349833+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:24:02.349833+0000","last_undegraded":"2024-08-17T17:24:02.349833+0000","last_fullsized":"2024-08-17T17:24:02.349833+0000","mapping_epoch":111,"log_start":"114'2600","ondisk_log_start":"114'2600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12663,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":144,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478883+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:23:56.478883+0000","last_peered":"2024-08-17T17:23:56.478883+0000","last_clean":"2024-08-17T17:23:56.478883+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:23:56.478883+0000","last_undegraded":"2024-08-17T17:23:56.478883+0000","last_fullsized":"2024-08-17T17:23:56.478883+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447824+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:23:56.447824+0000","last_peered":"2024-08-17T17:23:56.447824+0000","last_clean":"2024-08-17T17:23:56.447824+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:23:56.447824+0000","last_undegraded":"2024-08-17T17:23:56.447824+0000","last_fullsized":"2024-08-17T17:23:56.447824+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"154'12455","reported_seq":12558,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:02.358942+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:24:02.358942+0000","last_peered":"2024-08-17T17:24:02.358942+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:24:02.358942+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"114'2400","ondisk_log_start":"114'2400","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10055,"log_dups_size":0,"ondisk_log_size":10055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252116992,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12455,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-3131},{"shards":"0,2,5","objects":3847}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478932+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:23:56.478932+0000","last_peered":"2024-08-17T17:23:56.478932+0000","last_clean":"2024-08-17T17:23:56.478932+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:23:56.478932+0000","last_undegraded":"2024-08-17T17:23:56.478932+0000","last_fullsized":"2024-08-17T17:23:56.478932+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.473448+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:23:56.473448+0000","last_peered":"2024-08-17T17:23:56.473448+0000","last_clean":"2024-08-17T17:23:56.473448+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:23:56.473448+0000","last_undegraded":"2024-08-17T17:23:56.473448+0000","last_fullsized":"2024-08-17T17:23:56.473448+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"154'12503","reported_seq":13749,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.338983+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:24:02.338983+0000","last_peered":"2024-08-17T17:24:02.338983+0000","last_clean":"2024-08-17T17:24:02.338983+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:24:02.338983+0000","last_undegraded":"2024-08-17T17:24:02.338983+0000","last_fullsized":"2024-08-17T17:24:02.338983+0000","mapping_epoch":130,"log_start":"114'2500","ondisk_log_start":"114'2500","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10003,"log_dups_size":0,"ondisk_log_size":10003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260243456,"num_objects":3971,"num_object_clones":0,"num_object_copies":11913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12503,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.464199+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:23:57.464199+0000","last_peered":"2024-08-17T17:23:57.464199+0000","last_clean":"2024-08-17T17:23:57.464199+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:23:57.464199+0000","last_undegraded":"2024-08-17T17:23:57.464199+0000","last_fullsized":"2024-08-17T17:23:57.464199+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488650+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:23:57.488650+0000","last_peered":"2024-08-17T17:23:57.488650+0000","last_clean":"2024-08-17T17:23:57.488650+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:23:57.488650+0000","last_undegraded":"2024-08-17T17:23:57.488650+0000","last_fullsized":"2024-08-17T17:23:57.488650+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"154'12627","reported_seq":12724,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:02.393283+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:24:02.393283+0000","last_peered":"2024-08-17T17:24:02.393283+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:24:02.393283+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"114'2600","ondisk_log_start":"114'2600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256376856,"num_objects":3913,"num_object_clones":0,"num_object_copies":11739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3913,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12627,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":3913},{"shards":"2,7","objects":-3201}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":183,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467493+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:23:57.467493+0000","last_peered":"2024-08-17T17:23:57.467493+0000","last_clean":"2024-08-17T17:23:57.467493+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:23:57.467493+0000","last_undegraded":"2024-08-17T17:23:57.467493+0000","last_fullsized":"2024-08-17T17:23:57.467493+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447621+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:23:56.447621+0000","last_peered":"2024-08-17T17:23:56.447621+0000","last_clean":"2024-08-17T17:23:56.447621+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:23:56.447621+0000","last_undegraded":"2024-08-17T17:23:56.447621+0000","last_fullsized":"2024-08-17T17:23:56.447621+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"154'12733","reported_seq":12910,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:03.755272+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:24:03.755272+0000","last_peered":"2024-08-17T17:24:03.755272+0000","last_clean":"2024-08-17T17:24:03.755272+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:24:03.755272+0000","last_undegraded":"2024-08-17T17:24:03.755272+0000","last_fullsized":"2024-08-17T17:24:03.755272+0000","mapping_epoch":128,"log_start":"117'5756","ondisk_log_start":"117'5756","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257228800,"num_objects":3925,"num_object_clones":0,"num_object_copies":11775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12733,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":134,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.473547+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:23:56.473547+0000","last_peered":"2024-08-17T17:23:56.473547+0000","last_clean":"2024-08-17T17:23:56.473547+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:23:56.473547+0000","last_undegraded":"2024-08-17T17:23:56.473547+0000","last_fullsized":"2024-08-17T17:23:56.473547+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.448079+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:23:56.448079+0000","last_peered":"2024-08-17T17:23:56.448079+0000","last_clean":"2024-08-17T17:23:56.448079+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:23:56.448079+0000","last_undegraded":"2024-08-17T17:23:56.448079+0000","last_fullsized":"2024-08-17T17:23:56.448079+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"154'12819","reported_seq":17205,"reported_epoch":154,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:06.133324+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:24:06.133324+0000","last_peered":"2024-08-17T17:24:06.133324+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:24:06.133324+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"117'6100","ondisk_log_start":"117'6100","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253034496,"num_objects":3861,"num_object_clones":0,"num_object_copies":11583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1743,"num_objects_misplaced":3861,"num_objects_unfound":0,"num_objects_dirty":3861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12819,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2149,"num_bytes_recovered":105644032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":3861},{"shards":"4,6","objects":-1743}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447719+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:23:56.447719+0000","last_peered":"2024-08-17T17:23:56.447719+0000","last_clean":"2024-08-17T17:23:56.447719+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:23:56.447719+0000","last_undegraded":"2024-08-17T17:23:56.447719+0000","last_fullsized":"2024-08-17T17:23:56.447719+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.449362+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:23:56.449362+0000","last_peered":"2024-08-17T17:23:56.449362+0000","last_clean":"2024-08-17T17:23:56.449362+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:23:56.449362+0000","last_undegraded":"2024-08-17T17:23:56.449362+0000","last_fullsized":"2024-08-17T17:23:56.449362+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"154'12903","reported_seq":13054,"reported_epoch":154,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:06.173793+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:24:06.173793+0000","last_peered":"2024-08-17T17:24:06.173793+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:24:06.173793+0000","last_undegraded":"2024-08-17T17:24:06.173793+0000","last_fullsized":"2024-08-17T17:24:06.173793+0000","mapping_epoch":135,"log_start":"117'6141","ondisk_log_start":"117'6141","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6762,"log_dups_size":0,"ondisk_log_size":6762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254607360,"num_objects":3885,"num_object_clones":0,"num_object_copies":11655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3885,"num_objects_unfound":0,"num_objects_dirty":3885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12903,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":3885}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":163,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467119+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:23:57.467119+0000","last_peered":"2024-08-17T17:23:57.467119+0000","last_clean":"2024-08-17T17:23:57.467119+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:23:57.467119+0000","last_undegraded":"2024-08-17T17:23:57.467119+0000","last_fullsized":"2024-08-17T17:23:57.467119+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.465100+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:23:57.465100+0000","last_peered":"2024-08-17T17:23:57.465100+0000","last_clean":"2024-08-17T17:23:57.465100+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:23:57.465100+0000","last_undegraded":"2024-08-17T17:23:57.465100+0000","last_fullsized":"2024-08-17T17:23:57.465100+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"154'12693","reported_seq":25308,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:05.815978+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:24:05.815978+0000","last_peered":"2024-08-17T17:24:05.815978+0000","last_clean":"2024-08-17T17:24:05.815978+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:24:05.815978+0000","last_undegraded":"2024-08-17T17:24:05.815978+0000","last_fullsized":"2024-08-17T17:24:05.815978+0000","mapping_epoch":147,"log_start":"116'4520","ondisk_log_start":"116'4520","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246611968,"num_objects":3763,"num_object_clones":0,"num_object_copies":11289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12693,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":448,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.456922+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:23:57.456922+0000","last_peered":"2024-08-17T17:23:57.456922+0000","last_clean":"2024-08-17T17:23:57.456922+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:23:57.456922+0000","last_undegraded":"2024-08-17T17:23:57.456922+0000","last_fullsized":"2024-08-17T17:23:57.456922+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":678,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459402+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:23:57.459402+0000","last_peered":"2024-08-17T17:23:57.459402+0000","last_clean":"2024-08-17T17:23:57.459402+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:23:57.459402+0000","last_undegraded":"2024-08-17T17:23:57.459402+0000","last_fullsized":"2024-08-17T17:23:57.459402+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":120,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.463597+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:23:57.463597+0000","last_peered":"2024-08-17T17:23:57.463597+0000","last_clean":"2024-08-17T17:23:57.463597+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:23:57.463597+0000","last_undegraded":"2024-08-17T17:23:57.463597+0000","last_fullsized":"2024-08-17T17:23:57.463597+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.473775+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:23:56.473775+0000","last_peered":"2024-08-17T17:23:56.473775+0000","last_clean":"2024-08-17T17:23:56.473775+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:23:56.473775+0000","last_undegraded":"2024-08-17T17:23:56.473775+0000","last_fullsized":"2024-08-17T17:23:56.473775+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"154'12669","reported_seq":12747,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:03.693830+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:24:03.693830+0000","last_peered":"2024-08-17T17:24:03.693830+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:24:03.693830+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"116'5656","ondisk_log_start":"116'5656","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820928,"num_objects":3873,"num_object_clones":0,"num_object_copies":11619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12669,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-3237},{"shards":"0,1,5","objects":3873}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":525,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478391+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:23:56.478391+0000","last_peered":"2024-08-17T17:23:56.478391+0000","last_clean":"2024-08-17T17:23:56.478391+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:23:56.478391+0000","last_undegraded":"2024-08-17T17:23:56.478391+0000","last_fullsized":"2024-08-17T17:23:56.478391+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":168,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.487798+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:23:57.487798+0000","last_peered":"2024-08-17T17:23:57.487798+0000","last_clean":"2024-08-17T17:23:57.487798+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:23:57.487798+0000","last_undegraded":"2024-08-17T17:23:57.487798+0000","last_fullsized":"2024-08-17T17:23:57.487798+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.489044+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:23:57.489044+0000","last_peered":"2024-08-17T17:23:57.489044+0000","last_clean":"2024-08-17T17:23:57.489044+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:23:57.489044+0000","last_undegraded":"2024-08-17T17:23:57.489044+0000","last_fullsized":"2024-08-17T17:23:57.489044+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"154'12480","reported_seq":12581,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:02.330162+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:24:02.330162+0000","last_peered":"2024-08-17T17:24:02.330162+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:24:02.330162+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"114'2400","ondisk_log_start":"114'2400","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12480,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":3862},{"shards":"2,7","objects":-3211}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":564,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459480+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:23:57.459480+0000","last_peered":"2024-08-17T17:23:57.459480+0000","last_clean":"2024-08-17T17:23:57.459480+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:23:57.459480+0000","last_undegraded":"2024-08-17T17:23:57.459480+0000","last_fullsized":"2024-08-17T17:23:57.459480+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467188+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:23:57.467188+0000","last_peered":"2024-08-17T17:23:57.467188+0000","last_clean":"2024-08-17T17:23:57.467188+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:23:57.467188+0000","last_undegraded":"2024-08-17T17:23:57.467188+0000","last_fullsized":"2024-08-17T17:23:57.467188+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":185,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.470624+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:23:57.470624+0000","last_peered":"2024-08-17T17:23:57.470624+0000","last_clean":"2024-08-17T17:23:57.470624+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:23:57.470624+0000","last_undegraded":"2024-08-17T17:23:57.470624+0000","last_fullsized":"2024-08-17T17:23:57.470624+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"154'12405","reported_seq":13552,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:02.913422+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:24:02.913422+0000","last_peered":"2024-08-17T17:24:02.913422+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:24:02.913422+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"116'5071","ondisk_log_start":"116'5071","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252116992,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12405,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":3847}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":963,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:23:57.467167+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:23:57.467167+0000","last_peered":"2024-08-17T17:23:57.467167+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:23:57.467167+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447536+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:23:56.447536+0000","last_peered":"2024-08-17T17:23:56.447536+0000","last_clean":"2024-08-17T17:23:56.447536+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:23:56.447536+0000","last_undegraded":"2024-08-17T17:23:56.447536+0000","last_fullsized":"2024-08-17T17:23:56.447536+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":50,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.468044+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:23:57.468044+0000","last_peered":"2024-08-17T17:23:57.468044+0000","last_clean":"2024-08-17T17:23:57.468044+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:23:57.468044+0000","last_undegraded":"2024-08-17T17:23:57.468044+0000","last_fullsized":"2024-08-17T17:23:57.468044+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"154'12568","reported_seq":25371,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:03.032956+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:24:03.032956+0000","last_peered":"2024-08-17T17:24:03.032956+0000","last_clean":"2024-08-17T17:24:03.032956+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:24:03.032956+0000","last_undegraded":"2024-08-17T17:24:03.032956+0000","last_fullsized":"2024-08-17T17:24:03.032956+0000","mapping_epoch":147,"log_start":"116'5233","ondisk_log_start":"116'5233","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12568,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.464564+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:23:57.464564+0000","last_peered":"2024-08-17T17:23:57.464564+0000","last_clean":"2024-08-17T17:23:57.464564+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:23:57.464564+0000","last_undegraded":"2024-08-17T17:23:57.464564+0000","last_fullsized":"2024-08-17T17:23:57.464564+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":449,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.468483+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:23:57.468483+0000","last_peered":"2024-08-17T17:23:57.468483+0000","last_clean":"2024-08-17T17:23:57.468483+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:23:57.468483+0000","last_undegraded":"2024-08-17T17:23:57.468483+0000","last_fullsized":"2024-08-17T17:23:57.468483+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":155,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467605+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:23:57.467605+0000","last_peered":"2024-08-17T17:23:57.467605+0000","last_clean":"2024-08-17T17:23:57.467605+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:23:57.467605+0000","last_undegraded":"2024-08-17T17:23:57.467605+0000","last_fullsized":"2024-08-17T17:23:57.467605+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"154'12467","reported_seq":12613,"reported_epoch":154,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:05.691686+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:24:05.691686+0000","last_peered":"2024-08-17T17:24:05.691686+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:24:05.691686+0000","last_undegraded":"2024-08-17T17:24:05.691686+0000","last_fullsized":"2024-08-17T17:24:05.691686+0000","mapping_epoch":135,"log_start":"116'4288","ondisk_log_start":"116'4288","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8179,"log_dups_size":0,"ondisk_log_size":8179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249233408,"num_objects":3803,"num_object_clones":0,"num_object_copies":11409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3781,"num_objects_unfound":0,"num_objects_dirty":3803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12467,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":3803}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":152,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.468219+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:23:57.468219+0000","last_peered":"2024-08-17T17:23:57.468219+0000","last_clean":"2024-08-17T17:23:57.468219+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:23:57.468219+0000","last_undegraded":"2024-08-17T17:23:57.468219+0000","last_fullsized":"2024-08-17T17:23:57.468219+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":680,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:23:56.479041+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:23:56.479041+0000","last_peered":"2024-08-17T17:23:56.479041+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:23:56.479041+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":150,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488599+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:23:57.488599+0000","last_peered":"2024-08-17T17:23:57.488599+0000","last_clean":"2024-08-17T17:23:57.488599+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:23:57.488599+0000","last_undegraded":"2024-08-17T17:23:57.488599+0000","last_fullsized":"2024-08-17T17:23:57.488599+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"154'12774","reported_seq":12947,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:06.180617+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:24:06.180617+0000","last_peered":"2024-08-17T17:24:06.180617+0000","last_clean":"2024-08-17T17:24:06.180617+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:24:06.180617+0000","last_undegraded":"2024-08-17T17:24:06.180617+0000","last_fullsized":"2024-08-17T17:24:06.180617+0000","mapping_epoch":128,"log_start":"117'6075","ondisk_log_start":"117'6075","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243924992,"num_objects":3722,"num_object_clones":0,"num_object_copies":11166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12774,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":161,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.465328+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:23:57.465328+0000","last_peered":"2024-08-17T17:23:57.465328+0000","last_clean":"2024-08-17T17:23:57.465328+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:23:57.465328+0000","last_undegraded":"2024-08-17T17:23:57.465328+0000","last_fullsized":"2024-08-17T17:23:57.465328+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":643,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:23:57.466481+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:23:57.466481+0000","last_peered":"2024-08-17T17:23:57.466481+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:23:57.466481+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":158,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.466963+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:23:57.466963+0000","last_peered":"2024-08-17T17:23:57.466963+0000","last_clean":"2024-08-17T17:23:57.466963+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:23:57.466963+0000","last_undegraded":"2024-08-17T17:23:57.466963+0000","last_fullsized":"2024-08-17T17:23:57.466963+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"154'12687","reported_seq":14002,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:03.570042+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:24:03.570042+0000","last_peered":"2024-08-17T17:24:03.570042+0000","last_clean":"2024-08-17T17:24:03.570042+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:24:03.570042+0000","last_undegraded":"2024-08-17T17:24:03.570042+0000","last_fullsized":"2024-08-17T17:24:03.570042+0000","mapping_epoch":139,"log_start":"116'5130","ondisk_log_start":"116'5130","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12687,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":180,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488840+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:23:57.488840+0000","last_peered":"2024-08-17T17:23:57.488840+0000","last_clean":"2024-08-17T17:23:57.488840+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:23:57.488840+0000","last_undegraded":"2024-08-17T17:23:57.488840+0000","last_fullsized":"2024-08-17T17:23:57.488840+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":397,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.474251+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:23:56.474251+0000","last_peered":"2024-08-17T17:23:56.474251+0000","last_clean":"2024-08-17T17:23:56.474251+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:23:56.474251+0000","last_undegraded":"2024-08-17T17:23:56.474251+0000","last_fullsized":"2024-08-17T17:23:56.474251+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447945+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:23:56.447945+0000","last_peered":"2024-08-17T17:23:56.447945+0000","last_clean":"2024-08-17T17:23:56.447945+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:23:56.447945+0000","last_undegraded":"2024-08-17T17:23:56.447945+0000","last_fullsized":"2024-08-17T17:23:56.447945+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447597+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:23:56.447597+0000","last_peered":"2024-08-17T17:23:56.447597+0000","last_clean":"2024-08-17T17:23:56.447597+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:23:56.447597+0000","last_undegraded":"2024-08-17T17:23:56.447597+0000","last_fullsized":"2024-08-17T17:23:56.447597+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":47,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447546+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:23:56.447546+0000","last_peered":"2024-08-17T17:23:56.447546+0000","last_clean":"2024-08-17T17:23:56.447546+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:23:56.447546+0000","last_undegraded":"2024-08-17T17:23:56.447546+0000","last_fullsized":"2024-08-17T17:23:56.447546+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467428+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:23:57.467428+0000","last_peered":"2024-08-17T17:23:57.467428+0000","last_clean":"2024-08-17T17:23:57.467428+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:23:57.467428+0000","last_undegraded":"2024-08-17T17:23:57.467428+0000","last_fullsized":"2024-08-17T17:23:57.467428+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":148,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478784+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:23:56.478784+0000","last_peered":"2024-08-17T17:23:56.478784+0000","last_clean":"2024-08-17T17:23:56.478784+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:23:56.478784+0000","last_undegraded":"2024-08-17T17:23:56.478784+0000","last_fullsized":"2024-08-17T17:23:56.478784+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":82,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.478794+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:23:56.478794+0000","last_peered":"2024-08-17T17:23:56.478794+0000","last_clean":"2024-08-17T17:23:56.478794+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:23:56.478794+0000","last_undegraded":"2024-08-17T17:23:56.478794+0000","last_fullsized":"2024-08-17T17:23:56.478794+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":113,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.463179+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:23:57.463179+0000","last_peered":"2024-08-17T17:23:57.463179+0000","last_clean":"2024-08-17T17:23:57.463179+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:23:57.463179+0000","last_undegraded":"2024-08-17T17:23:57.463179+0000","last_fullsized":"2024-08-17T17:23:57.463179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":160,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.489196+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:23:57.489196+0000","last_peered":"2024-08-17T17:23:57.489196+0000","last_clean":"2024-08-17T17:23:57.489196+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:23:57.489196+0000","last_undegraded":"2024-08-17T17:23:57.489196+0000","last_fullsized":"2024-08-17T17:23:57.489196+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":162,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.466891+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:23:57.466891+0000","last_peered":"2024-08-17T17:23:57.466891+0000","last_clean":"2024-08-17T17:23:57.466891+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:23:57.466891+0000","last_undegraded":"2024-08-17T17:23:57.466891+0000","last_fullsized":"2024-08-17T17:23:57.466891+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.474177+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:23:56.474177+0000","last_peered":"2024-08-17T17:23:56.474177+0000","last_clean":"2024-08-17T17:23:56.474177+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:23:56.474177+0000","last_undegraded":"2024-08-17T17:23:56.474177+0000","last_fullsized":"2024-08-17T17:23:56.474177+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":178,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467033+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:23:57.467033+0000","last_peered":"2024-08-17T17:23:57.467033+0000","last_clean":"2024-08-17T17:23:57.467033+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:23:57.467033+0000","last_undegraded":"2024-08-17T17:23:57.467033+0000","last_fullsized":"2024-08-17T17:23:57.467033+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.489001+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:23:57.489001+0000","last_peered":"2024-08-17T17:23:57.489001+0000","last_clean":"2024-08-17T17:23:57.489001+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:23:57.489001+0000","last_undegraded":"2024-08-17T17:23:57.489001+0000","last_fullsized":"2024-08-17T17:23:57.489001+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":158,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.467028+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:23:57.467028+0000","last_peered":"2024-08-17T17:23:57.467028+0000","last_clean":"2024-08-17T17:23:57.467028+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:23:57.467028+0000","last_undegraded":"2024-08-17T17:23:57.467028+0000","last_fullsized":"2024-08-17T17:23:57.467028+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":175,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.464395+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:23:57.464395+0000","last_peered":"2024-08-17T17:23:57.464395+0000","last_clean":"2024-08-17T17:23:57.464395+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:23:57.464395+0000","last_undegraded":"2024-08-17T17:23:57.464395+0000","last_fullsized":"2024-08-17T17:23:57.464395+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.448808+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:23:56.448808+0000","last_peered":"2024-08-17T17:23:56.448808+0000","last_clean":"2024-08-17T17:23:56.448808+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:23:56.448808+0000","last_undegraded":"2024-08-17T17:23:56.448808+0000","last_fullsized":"2024-08-17T17:23:56.448808+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":183,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459692+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:23:57.459692+0000","last_peered":"2024-08-17T17:23:57.459692+0000","last_clean":"2024-08-17T17:23:57.459692+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:23:57.459692+0000","last_undegraded":"2024-08-17T17:23:57.459692+0000","last_fullsized":"2024-08-17T17:23:57.459692+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":160,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.488535+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:23:57.488535+0000","last_peered":"2024-08-17T17:23:57.488535+0000","last_clean":"2024-08-17T17:23:57.488535+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:23:57.488535+0000","last_undegraded":"2024-08-17T17:23:57.488535+0000","last_fullsized":"2024-08-17T17:23:57.488535+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447460+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:23:56.447460+0000","last_peered":"2024-08-17T17:23:56.447460+0000","last_clean":"2024-08-17T17:23:56.447460+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:23:56.447460+0000","last_undegraded":"2024-08-17T17:23:56.447460+0000","last_fullsized":"2024-08-17T17:23:56.447460+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":138,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447454+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:23:56.447454+0000","last_peered":"2024-08-17T17:23:56.447454+0000","last_clean":"2024-08-17T17:23:56.447454+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:23:56.447454+0000","last_undegraded":"2024-08-17T17:23:56.447454+0000","last_fullsized":"2024-08-17T17:23:56.447454+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.448712+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:23:56.448712+0000","last_peered":"2024-08-17T17:23:56.448712+0000","last_clean":"2024-08-17T17:23:56.448712+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:23:56.448712+0000","last_undegraded":"2024-08-17T17:23:56.448712+0000","last_fullsized":"2024-08-17T17:23:56.448712+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":123,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447908+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:23:56.447908+0000","last_peered":"2024-08-17T17:23:56.447908+0000","last_clean":"2024-08-17T17:23:56.447908+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:23:56.447908+0000","last_undegraded":"2024-08-17T17:23:56.447908+0000","last_fullsized":"2024-08-17T17:23:56.447908+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":180,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.463706+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:23:57.463706+0000","last_peered":"2024-08-17T17:23:57.463706+0000","last_clean":"2024-08-17T17:23:57.463706+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:23:57.463706+0000","last_undegraded":"2024-08-17T17:23:57.463706+0000","last_fullsized":"2024-08-17T17:23:57.463706+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":168,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.465014+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:23:57.465014+0000","last_peered":"2024-08-17T17:23:57.465014+0000","last_clean":"2024-08-17T17:23:57.465014+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:23:57.465014+0000","last_undegraded":"2024-08-17T17:23:57.465014+0000","last_fullsized":"2024-08-17T17:23:57.465014+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":37,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.447167+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:23:56.447167+0000","last_peered":"2024-08-17T17:23:56.447167+0000","last_clean":"2024-08-17T17:23:56.447167+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:23:56.447167+0000","last_undegraded":"2024-08-17T17:23:56.447167+0000","last_fullsized":"2024-08-17T17:23:56.447167+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":171,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.489099+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:23:57.489099+0000","last_peered":"2024-08-17T17:23:57.489099+0000","last_clean":"2024-08-17T17:23:57.489099+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:23:57.489099+0000","last_undegraded":"2024-08-17T17:23:57.489099+0000","last_fullsized":"2024-08-17T17:23:57.489099+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4047044632,"num_objects":61754,"num_object_clones":0,"num_object_copies":185262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34856,"num_objects_misplaced":11527,"num_objects_unfound":0,"num_objects_dirty":61754,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202130,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17158,"num_bytes_recovered":958857216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13189001216,"data_stored":13188989000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131160,"ondisk_log_size":131160,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201527,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1337532,"kb_used_data":1212084,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":92407620,"statfs":{"total":95995035648,"available":94625402880,"internally_reserved":0,"allocated":1241174016,"data_stored":1240291421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":128448971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463165,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070576,"kb_used_data":981224,"kb_used_omap":1,"kb_used_meta":89342,"kb_avail":92674576,"statfs":{"total":95995035648,"available":94898765824,"internally_reserved":0,"allocated":1004773376,"data_stored":1003903235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":91486663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724801,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986437,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2151728,"kb_used_data":1972136,"kb_used_omap":1,"kb_used_meta":179582,"kb_avail":91593424,"statfs":{"total":95995035648,"available":93791666176,"internally_reserved":0,"allocated":2019467264,"data_stored":2018596327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":183892422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435328,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1121312,"kb_used_data":1013272,"kb_used_omap":5,"kb_used_meta":108026,"kb_avail":92623840,"statfs":{"total":95995035648,"available":94846812160,"internally_reserved":0,"allocated":1037590528,"data_stored":1036731245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":110618697},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598854,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1647652,"kb_used_data":1499356,"kb_used_omap":7,"kb_used_meta":148280,"kb_avail":92097500,"statfs":{"total":95995035648,"available":94307840000,"internally_reserved":0,"allocated":1535340544,"data_stored":1534477645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":151838893},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762381,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1361048,"kb_used_data":1253008,"kb_used_omap":7,"kb_used_meta":108024,"kb_avail":92384104,"statfs":{"total":95995035648,"available":94601322496,"internally_reserved":0,"allocated":1283080192,"data_stored":1282228687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":110616993},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925907,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2445352,"kb_used_data":2269216,"kb_used_omap":7,"kb_used_meta":176056,"kb_avail":91299800,"statfs":{"total":95995035648,"available":93490995200,"internally_reserved":0,"allocated":2323677184,"data_stored":2322810970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":180281948},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2299527168,"data_stored":2299527168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1266548736,"data_stored":1266548736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1529352192,"data_stored":1529348120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1025114112,"data_stored":1025114112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2008416256,"data_stored":2008416256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2830307328,"data_stored":2830303256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1000996864,"data_stored":1000996864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1228738560,"data_stored":1228734488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:08.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:08.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:08.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:08.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:08.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:08.611 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:08.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:08.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.803 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:08.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:08.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:08 smithi175 ceph-mon[101434]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 34886/185346 objects degraded (18.822%); 11527/185346 objects misplaced (6.219%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:08 smithi175 ceph-mon[101434]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34886/185346 objects degraded (18.822%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:08.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[104516]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 34886/185346 objects degraded (18.822%); 11527/185346 objects misplaced (6.219%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:08.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[104516]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:08.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34886/185346 objects degraded (18.822%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[109246]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 34886/185346 objects degraded (18.822%); 11527/185346 objects misplaced (6.219%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[109246]: from='client.54610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34886/185346 objects degraded (18.822%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:08.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:09.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:09.065 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:09.124 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:09.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:09.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:09.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:09.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.524 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:09.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:09.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:09.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:09.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:09.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:09.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:10.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:10.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:10.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:10.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:10.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:10.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:10.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:10.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:10.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:10.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:10.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:10.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:10 smithi175 ceph-mon[101434]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 34886/179424 objects degraded (19.443%); 11527/179424 objects misplaced (6.424%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[104516]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 34886/179424 objects degraded (19.443%); 11527/179424 objects misplaced (6.424%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[109246]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 34886/179424 objects degraded (19.443%); 11527/179424 objects misplaced (6.424%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:11.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:11.051 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:11.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.190 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:11.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:11.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:11.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:11.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:11.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:11.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:11.598 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:11.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:11.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:11.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:11.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:11.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:11.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:11.999 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:12.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:12.129 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:12.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:12.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:12.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:12.376 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:12.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:12.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:12.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:12.666 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:12.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:12.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:12 smithi064 ceph-mon[104516]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 34886/178830 objects degraded (19.508%); 11332/178830 objects misplaced (6.337%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:24:12.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:12 smithi064 ceph-mon[109246]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 34886/178830 objects degraded (19.508%); 11332/178830 objects misplaced (6.337%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:24:12.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:12.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:12.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:12 smithi175 ceph-mon[101434]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 34886/178830 objects degraded (19.508%); 11332/178830 objects misplaced (6.337%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:24:12.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:12.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:12.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:24:12.974 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:12.995 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:24:12.995 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:13.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:24:13.065 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:13.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:13.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:13.118 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:24:13.118 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:13.162 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:24:13.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:13.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:13.209 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:24:13.210 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:13.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:13.255 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:24:13.255 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:13.286 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:24:13.286 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:13.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:13.464 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:13.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:13.512 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:24:13.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:13.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:13.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4244285192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34886/178830 objects degraded (19.508%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4244285192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34886/178830 objects degraded (19.508%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:13.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:13.900 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:13.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4244285192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:13.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34886/178830 objects degraded (19.508%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:13.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:14.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:14.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:14.109 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:13] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-08-17T17:24:14.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:14.190 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:14.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:14.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:14.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:14.455 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:14.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:14.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:14.622 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:24:14.622 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:24:14.622 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:24:14.622 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:24:14.622 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:24:14.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:14.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:14.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:14.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:14 smithi175 ceph-mon[101434]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 34986/172665 objects degraded (20.262%); 10920/172665 objects misplaced (6.324%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:24:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:14 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:14 smithi175 ceph-mon[101434]: osdmap e155: 8 total, 8 up, 7 in 2024-08-17T17:24:14.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[104516]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 34986/172665 objects degraded (20.262%); 10920/172665 objects misplaced (6.324%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[104516]: osdmap e155: 8 total, 8 up, 7 in 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[109246]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 34986/172665 objects degraded (20.262%); 10920/172665 objects misplaced (6.324%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/337027047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:14 smithi064 ceph-mon[109246]: osdmap e155: 8 total, 8 up, 7 in 2024-08-17T17:24:14.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:14.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:15.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:15.107 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:15.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:15.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:15.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:15.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:15.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:15.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:15.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:15.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:15.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:15.802 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:15.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:15.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.953 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:15.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:16.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:16.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:16.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:16.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:16.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:16.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:16.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:16.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:16.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:16.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:24:16.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:16.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:16.729 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:16.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:16.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:16.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:16 smithi064 ceph-mon[109246]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 34986/169578 objects degraded (20.631%); 10920/169578 objects misplaced (6.440%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:24:16.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:16 smithi064 ceph-mon[104516]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 34986/169578 objects degraded (20.631%); 10920/169578 objects misplaced (6.440%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:24:16.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:16.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:16 smithi175 ceph-mon[101434]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 34986/169578 objects degraded (20.631%); 10920/169578 objects misplaced (6.440%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:24:17.010 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:17.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:17.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:17.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.215 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:17.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:17.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:17.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:17.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:17.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:17.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:17.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:24:17.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:17.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:17.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:17.781 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:17.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:17.906 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:18.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:18.011 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:18.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:18.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:18.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:18.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:18.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 35105/165969 objects degraded (21.152%); 10268/165969 objects misplaced (6.187%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35105/165969 objects degraded (21.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 35105/165969 objects degraded (21.152%); 10268/165969 objects misplaced (6.187%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35105/165969 objects degraded (21.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 35105/165969 objects degraded (21.152%); 10268/165969 objects misplaced (6.187%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:24:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:24:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:18.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35105/165969 objects degraded (21.152%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:18.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:18.470 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:18.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:18.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:18.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:18.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:18.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:18.802 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:18.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:19.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:19.063 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:19.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:19.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:19.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:19.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:19.301 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:19.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:19.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:19.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:19.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:19.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:19.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:19.744 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:19.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:19.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:19.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:20.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:20.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:20.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:20.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:20.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:20 smithi175 ceph-mon[101434]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35105/160008 objects degraded (21.940%); 10268/160008 objects misplaced (6.417%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:20.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:20 smithi064 ceph-mon[104516]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35105/160008 objects degraded (21.940%); 10268/160008 objects misplaced (6.417%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:20.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:20 smithi064 ceph-mon[109246]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.3 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35105/160008 objects degraded (21.940%); 10268/160008 objects misplaced (6.417%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:20.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:20.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:20.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:20.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:20.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:20.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:20.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:20.910 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:20.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:21.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:21.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:21.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:21.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:21.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:21.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:21.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:21.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:21.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:21.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:21.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:21.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:21.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:21.766 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:24:21.767 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:24:21.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:21.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:21.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:22.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:22.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:22.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:22.280 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:22.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:22.382 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:22.411 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:22 smithi064 ceph-mon[104516]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35105/159402 objects degraded (22.023%); 10067/159402 objects misplaced (6.315%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:22 smithi064 ceph-mon[109246]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35105/159402 objects degraded (22.023%); 10067/159402 objects misplaced (6.315%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:22.513 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:22.535 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:22.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:22 smithi175 ceph-mon[101434]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35105/159402 objects degraded (22.023%); 10067/159402 objects misplaced (6.315%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:22.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:22.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:22.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:22.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:22.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:23.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:23.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:23.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:23.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:23.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:23.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:23.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:23.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:23 smithi064 ceph-mon[104516]: from='client.44813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:23.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35105/159402 objects degraded (22.023%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:23.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:23 smithi064 ceph-mon[109246]: from='client.44813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:23.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35105/159402 objects degraded (22.023%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:23.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:23.484 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:23.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:23.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:23 smithi175 ceph-mon[101434]: from='client.44813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35105/159402 objects degraded (22.023%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:23.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:23.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:23.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:23.887 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:23.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.055 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:24.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:24.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:24.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:24.202 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:23] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-08-17T17:24:24.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:24 smithi064 ceph-mon[109246]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35209/153225 objects degraded (22.979%); 9610/153225 objects misplaced (6.272%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:24.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:24 smithi064 ceph-mon[104516]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35209/153225 objects degraded (22.979%); 9610/153225 objects misplaced (6.272%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:24.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:24.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:24.438 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:24.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:24.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:24.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:24 smithi175 ceph-mon[101434]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 35209/153225 objects degraded (22.979%); 9610/153225 objects misplaced (6.272%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:24.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:24.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:24.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:24.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:24.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:24.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:25.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:25.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:25.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:25.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:25.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:25.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:25.550 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":456,"stamp":"2024-08-17T17:24:20.835979+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3485079549,"num_objects":53134,"num_object_clones":0,"num_object_copies":159402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35105,"num_objects_misplaced":10067,"num_objects_unfound":0,"num_objects_dirty":53134,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":211901,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17425,"num_bytes_recovered":977673380,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132390,"ondisk_log_size":132390,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9712720,"kb_used_data":8778776,"kb_used_omap":33,"kb_used_meta":933662,"kb_avail":646503344,"statfs":{"total":671965249536,"available":662019424256,"internally_reserved":0,"allocated":8989466624,"data_stored":8983348044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":956070475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,4,6,7,11,6,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-424411136,"num_objects":-6476,"num_object_clones":0,"num_object_copies":-19428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":219,"num_objects_misplaced":-1265,"num_objects_unfound":0,"num_objects_dirty":-6476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6476,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":196,"num_bytes_recovered":12845056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006423"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":164,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.542688+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:24:13.542688+0000","last_peered":"2024-08-17T17:24:13.542688+0000","last_clean":"2024-08-17T17:24:13.542688+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:24:13.542688+0000","last_undegraded":"2024-08-17T17:24:13.542688+0000","last_fullsized":"2024-08-17T17:24:13.542688+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":149,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551072+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:24:13.551072+0000","last_peered":"2024-08-17T17:24:13.551072+0000","last_clean":"2024-08-17T17:24:13.551072+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:24:13.551072+0000","last_undegraded":"2024-08-17T17:24:13.551072+0000","last_fullsized":"2024-08-17T17:24:13.551072+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.457344+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:23:57.457344+0000","last_peered":"2024-08-17T17:23:57.457344+0000","last_clean":"2024-08-17T17:23:57.457344+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:23:57.457344+0000","last_undegraded":"2024-08-17T17:23:57.457344+0000","last_fullsized":"2024-08-17T17:23:57.457344+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.524203+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:24:13.524203+0000","last_peered":"2024-08-17T17:24:13.524203+0000","last_clean":"2024-08-17T17:24:13.524203+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:24:13.524203+0000","last_undegraded":"2024-08-17T17:24:13.524203+0000","last_fullsized":"2024-08-17T17:24:13.524203+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":155,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523166+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:24:13.523166+0000","last_peered":"2024-08-17T17:24:13.523166+0000","last_clean":"2024-08-17T17:24:13.523166+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:24:13.523166+0000","last_undegraded":"2024-08-17T17:24:13.523166+0000","last_fullsized":"2024-08-17T17:24:13.523166+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":120,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551239+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:24:13.551239+0000","last_peered":"2024-08-17T17:24:13.551239+0000","last_clean":"2024-08-17T17:24:13.551239+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:24:13.551239+0000","last_undegraded":"2024-08-17T17:24:13.551239+0000","last_fullsized":"2024-08-17T17:24:13.551239+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.542633+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:24:13.542633+0000","last_peered":"2024-08-17T17:24:13.542633+0000","last_clean":"2024-08-17T17:24:13.542633+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:24:13.542633+0000","last_undegraded":"2024-08-17T17:24:13.542633+0000","last_fullsized":"2024-08-17T17:24:13.542633+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":144,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.535455+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:24:13.535455+0000","last_peered":"2024-08-17T17:24:13.535455+0000","last_clean":"2024-08-17T17:24:13.535455+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:24:13.535455+0000","last_undegraded":"2024-08-17T17:24:13.535455+0000","last_fullsized":"2024-08-17T17:24:13.535455+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"155'13063","reported_seq":14234,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.797174+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:24:15.797174+0000","last_peered":"2024-08-17T17:24:15.797174+0000","last_clean":"2024-08-17T17:24:15.797174+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:24:15.797174+0000","last_undegraded":"2024-08-17T17:24:15.797174+0000","last_fullsized":"2024-08-17T17:24:15.797174+0000","mapping_epoch":135,"log_start":"116'4900","ondisk_log_start":"116'4900","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219611136,"num_objects":3351,"num_object_clones":0,"num_object_copies":10053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13063,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":144,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.456903+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:23:57.456903+0000","last_peered":"2024-08-17T17:23:57.456903+0000","last_clean":"2024-08-17T17:23:57.456903+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:23:57.456903+0000","last_undegraded":"2024-08-17T17:23:57.456903+0000","last_fullsized":"2024-08-17T17:23:57.456903+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":32,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.456734+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:23:57.456734+0000","last_peered":"2024-08-17T17:23:57.456734+0000","last_clean":"2024-08-17T17:23:57.456734+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:23:57.456734+0000","last_undegraded":"2024-08-17T17:23:57.456734+0000","last_fullsized":"2024-08-17T17:23:57.456734+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"155'13279","reported_seq":13379,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.498648+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:24:17.498648+0000","last_peered":"2024-08-17T17:24:17.498648+0000","last_clean":"2024-08-17T17:24:17.498648+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:24:17.498648+0000","last_undegraded":"2024-08-17T17:24:17.498648+0000","last_fullsized":"2024-08-17T17:24:17.498648+0000","mapping_epoch":111,"log_start":"115'3200","ondisk_log_start":"115'3200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10079,"log_dups_size":0,"ondisk_log_size":10079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211222528,"num_objects":3223,"num_object_clones":0,"num_object_copies":9669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13279,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":145,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.458244+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:23:57.458244+0000","last_peered":"2024-08-17T17:23:57.458244+0000","last_clean":"2024-08-17T17:23:57.458244+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:23:57.458244+0000","last_undegraded":"2024-08-17T17:23:57.458244+0000","last_fullsized":"2024-08-17T17:23:57.458244+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459797+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:23:57.459797+0000","last_peered":"2024-08-17T17:23:57.459797+0000","last_clean":"2024-08-17T17:23:57.459797+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:23:57.459797+0000","last_undegraded":"2024-08-17T17:23:57.459797+0000","last_fullsized":"2024-08-17T17:23:57.459797+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"155'13056","reported_seq":13161,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:17.518334+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:24:17.518334+0000","last_peered":"2024-08-17T17:24:17.518334+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:24:17.518334+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"115'3000","ondisk_log_start":"115'3000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13056,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-3732},{"shards":"0,2,5","objects":3246}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":116,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.457536+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:23:57.457536+0000","last_peered":"2024-08-17T17:23:57.457536+0000","last_clean":"2024-08-17T17:23:57.457536+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:23:57.457536+0000","last_undegraded":"2024-08-17T17:23:57.457536+0000","last_fullsized":"2024-08-17T17:23:57.457536+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":116,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.497665+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:23:57.497665+0000","last_peered":"2024-08-17T17:23:57.497665+0000","last_clean":"2024-08-17T17:23:57.497665+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:23:57.497665+0000","last_undegraded":"2024-08-17T17:23:57.497665+0000","last_fullsized":"2024-08-17T17:23:57.497665+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"155'13098","reported_seq":14346,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.528400+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:24:17.528400+0000","last_peered":"2024-08-17T17:24:17.528400+0000","last_clean":"2024-08-17T17:24:17.528400+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:24:17.528400+0000","last_undegraded":"2024-08-17T17:24:17.528400+0000","last_fullsized":"2024-08-17T17:24:17.528400+0000","mapping_epoch":130,"log_start":"115'3000","ondisk_log_start":"115'3000","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221249536,"num_objects":3376,"num_object_clones":0,"num_object_copies":10128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13098,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.542819+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:24:13.542819+0000","last_peered":"2024-08-17T17:24:13.542819+0000","last_clean":"2024-08-17T17:24:13.542819+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:24:13.542819+0000","last_undegraded":"2024-08-17T17:24:13.542819+0000","last_fullsized":"2024-08-17T17:24:13.542819+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":170,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.524420+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:24:13.524420+0000","last_peered":"2024-08-17T17:24:13.524420+0000","last_clean":"2024-08-17T17:24:13.524420+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:24:13.524420+0000","last_undegraded":"2024-08-17T17:24:13.524420+0000","last_fullsized":"2024-08-17T17:24:13.524420+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"155'13201","reported_seq":13306,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:17.452304+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:24:17.452304+0000","last_peered":"2024-08-17T17:24:17.452304+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:24:17.452304+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"115'3200","ondisk_log_start":"115'3200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218759192,"num_objects":3339,"num_object_clones":0,"num_object_copies":10017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3339,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13201,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":3339},{"shards":"2,7","objects":-3775}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":185,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.545251+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:24:13.545251+0000","last_peered":"2024-08-17T17:24:13.545251+0000","last_clean":"2024-08-17T17:24:13.545251+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:24:13.545251+0000","last_undegraded":"2024-08-17T17:24:13.545251+0000","last_fullsized":"2024-08-17T17:24:13.545251+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460536+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:23:57.460536+0000","last_peered":"2024-08-17T17:23:57.460536+0000","last_clean":"2024-08-17T17:23:57.460536+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:23:57.460536+0000","last_undegraded":"2024-08-17T17:23:57.460536+0000","last_fullsized":"2024-08-17T17:23:57.460536+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"155'13351","reported_seq":13530,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:18.760140+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:24:18.760140+0000","last_peered":"2024-08-17T17:24:18.760140+0000","last_clean":"2024-08-17T17:24:18.760140+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:24:18.760140+0000","last_undegraded":"2024-08-17T17:24:18.760140+0000","last_fullsized":"2024-08-17T17:24:18.760140+0000","mapping_epoch":128,"log_start":"117'6356","ondisk_log_start":"117'6356","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216727552,"num_objects":3307,"num_object_clones":0,"num_object_copies":9921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13351,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":135,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.497788+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:23:57.497788+0000","last_peered":"2024-08-17T17:23:57.497788+0000","last_clean":"2024-08-17T17:23:57.497788+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:23:57.497788+0000","last_undegraded":"2024-08-17T17:23:57.497788+0000","last_fullsized":"2024-08-17T17:23:57.497788+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460870+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:23:57.460870+0000","last_peered":"2024-08-17T17:23:57.460870+0000","last_clean":"2024-08-17T17:23:57.460870+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:23:57.460870+0000","last_undegraded":"2024-08-17T17:23:57.460870+0000","last_fullsized":"2024-08-17T17:23:57.460870+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"155'13233","reported_seq":18012,"reported_epoch":155,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:16.196966+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:24:16.196966+0000","last_peered":"2024-08-17T17:24:16.196966+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:24:16.196966+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"117'6500","ondisk_log_start":"117'6500","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225902592,"num_objects":3447,"num_object_clones":0,"num_object_copies":10341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1962,"num_objects_misplaced":3446,"num_objects_unfound":0,"num_objects_dirty":3447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13233,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2345,"num_bytes_recovered":118489088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":3446},{"shards":"4,6","objects":-1961}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459687+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:23:57.459687+0000","last_peered":"2024-08-17T17:23:57.459687+0000","last_clean":"2024-08-17T17:23:57.459687+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:23:57.459687+0000","last_undegraded":"2024-08-17T17:23:57.459687+0000","last_fullsized":"2024-08-17T17:23:57.459687+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.463501+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:23:57.463501+0000","last_peered":"2024-08-17T17:23:57.463501+0000","last_clean":"2024-08-17T17:23:57.463501+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:23:57.463501+0000","last_undegraded":"2024-08-17T17:23:57.463501+0000","last_fullsized":"2024-08-17T17:23:57.463501+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"155'13339","reported_seq":13491,"reported_epoch":155,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:16.152776+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:24:16.152776+0000","last_peered":"2024-08-17T17:24:16.152776+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:24:16.152776+0000","last_undegraded":"2024-08-17T17:24:16.152776+0000","last_fullsized":"2024-08-17T17:24:16.152776+0000","mapping_epoch":135,"log_start":"118'6641","ondisk_log_start":"118'6641","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226033664,"num_objects":3449,"num_object_clones":0,"num_object_copies":10347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3449,"num_objects_unfound":0,"num_objects_dirty":3449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13339,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":3449}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":165,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.550918+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:24:13.550918+0000","last_peered":"2024-08-17T17:24:13.550918+0000","last_clean":"2024-08-17T17:24:13.550918+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:24:13.550918+0000","last_undegraded":"2024-08-17T17:24:13.550918+0000","last_fullsized":"2024-08-17T17:24:13.550918+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":152,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.584293+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:24:13.584293+0000","last_peered":"2024-08-17T17:24:13.584293+0000","last_clean":"2024-08-17T17:24:13.584293+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:24:13.584293+0000","last_undegraded":"2024-08-17T17:24:13.584293+0000","last_fullsized":"2024-08-17T17:24:13.584293+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"155'13083","reported_seq":25699,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.806659+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:24:15.806659+0000","last_peered":"2024-08-17T17:24:15.806659+0000","last_clean":"2024-08-17T17:24:15.806659+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:24:15.806659+0000","last_undegraded":"2024-08-17T17:24:15.806659+0000","last_fullsized":"2024-08-17T17:24:15.806659+0000","mapping_epoch":147,"log_start":"116'4920","ondisk_log_start":"116'4920","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221052928,"num_objects":3373,"num_object_clones":0,"num_object_copies":10119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13083,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":449,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.860819+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:24:00.860819+0000","last_peered":"2024-08-17T17:24:00.860819+0000","last_clean":"2024-08-17T17:24:00.860819+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:24:00.860819+0000","last_undegraded":"2024-08-17T17:24:00.860819+0000","last_fullsized":"2024-08-17T17:24:00.860819+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":680,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.561345+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:24:13.561345+0000","last_peered":"2024-08-17T17:24:13.561345+0000","last_clean":"2024-08-17T17:24:13.561345+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:24:13.561345+0000","last_undegraded":"2024-08-17T17:24:13.561345+0000","last_fullsized":"2024-08-17T17:24:13.561345+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":122,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.545478+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:24:13.545478+0000","last_peered":"2024-08-17T17:24:13.545478+0000","last_clean":"2024-08-17T17:24:13.545478+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:24:13.545478+0000","last_undegraded":"2024-08-17T17:24:13.545478+0000","last_fullsized":"2024-08-17T17:24:13.545478+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.497657+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:23:57.497657+0000","last_peered":"2024-08-17T17:23:57.497657+0000","last_clean":"2024-08-17T17:23:57.497657+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:23:57.497657+0000","last_undegraded":"2024-08-17T17:23:57.497657+0000","last_fullsized":"2024-08-17T17:23:57.497657+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"155'13288","reported_seq":13368,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:18.795445+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:24:18.795445+0000","last_peered":"2024-08-17T17:24:18.795445+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:24:18.795445+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"117'6256","ondisk_log_start":"117'6256","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213254144,"num_objects":3254,"num_object_clones":0,"num_object_copies":9762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13288,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-3856},{"shards":"0,1,5","objects":3254}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":526,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.456818+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:23:57.456818+0000","last_peered":"2024-08-17T17:23:57.456818+0000","last_clean":"2024-08-17T17:23:57.456818+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:23:57.456818+0000","last_undegraded":"2024-08-17T17:23:57.456818+0000","last_fullsized":"2024-08-17T17:23:57.456818+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.522796+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:24:13.522796+0000","last_peered":"2024-08-17T17:24:13.522796+0000","last_clean":"2024-08-17T17:24:13.522796+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:24:13.522796+0000","last_undegraded":"2024-08-17T17:24:13.522796+0000","last_fullsized":"2024-08-17T17:24:13.522796+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":167,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523349+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:24:13.523349+0000","last_peered":"2024-08-17T17:24:13.523349+0000","last_clean":"2024-08-17T17:24:13.523349+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:24:13.523349+0000","last_undegraded":"2024-08-17T17:24:13.523349+0000","last_fullsized":"2024-08-17T17:24:13.523349+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"155'13064","reported_seq":13167,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:17.537668+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:24:17.537668+0000","last_peered":"2024-08-17T17:24:17.537668+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:24:17.537668+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"115'3000","ondisk_log_start":"115'3000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214827008,"num_objects":3278,"num_object_clones":0,"num_object_copies":9834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13064,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":3278},{"shards":"2,7","objects":-3795}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":566,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.560745+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:24:13.560745+0000","last_peered":"2024-08-17T17:24:13.560745+0000","last_clean":"2024-08-17T17:24:13.560745+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:24:13.560745+0000","last_undegraded":"2024-08-17T17:24:13.560745+0000","last_fullsized":"2024-08-17T17:24:13.560745+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":147,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551514+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:24:13.551514+0000","last_peered":"2024-08-17T17:24:13.551514+0000","last_clean":"2024-08-17T17:24:13.551514+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:24:13.551514+0000","last_undegraded":"2024-08-17T17:24:13.551514+0000","last_fullsized":"2024-08-17T17:24:13.551514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":188,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.540670+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:24:13.540670+0000","last_peered":"2024-08-17T17:24:13.540670+0000","last_clean":"2024-08-17T17:24:13.540670+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:24:13.540670+0000","last_undegraded":"2024-08-17T17:24:13.540670+0000","last_fullsized":"2024-08-17T17:24:13.540670+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"155'13011","reported_seq":14160,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:18.034315+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:24:18.034315+0000","last_peered":"2024-08-17T17:24:18.034315+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:24:18.034315+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"117'5671","ondisk_log_start":"117'5671","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7340,"log_dups_size":0,"ondisk_log_size":7340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13011,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":3241}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":965,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:13.536474+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:24:13.536474+0000","last_peered":"2024-08-17T17:24:13.536474+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:24:13.536474+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460250+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:23:57.460250+0000","last_peered":"2024-08-17T17:23:57.460250+0000","last_clean":"2024-08-17T17:23:57.460250+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:23:57.460250+0000","last_undegraded":"2024-08-17T17:23:57.460250+0000","last_fullsized":"2024-08-17T17:23:57.460250+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.549383+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:24:13.549383+0000","last_peered":"2024-08-17T17:24:13.549383+0000","last_clean":"2024-08-17T17:24:13.549383+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:24:13.549383+0000","last_undegraded":"2024-08-17T17:24:13.549383+0000","last_fullsized":"2024-08-17T17:24:13.549383+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"155'13168","reported_seq":25973,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.896114+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:24:17.896114+0000","last_peered":"2024-08-17T17:24:17.896114+0000","last_clean":"2024-08-17T17:24:17.896114+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:24:17.896114+0000","last_undegraded":"2024-08-17T17:24:17.896114+0000","last_fullsized":"2024-08-17T17:24:17.896114+0000","mapping_epoch":147,"log_start":"117'5833","ondisk_log_start":"117'5833","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220987392,"num_objects":3372,"num_object_clones":0,"num_object_copies":10116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13168,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":129,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.543166+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:24:13.543166+0000","last_peered":"2024-08-17T17:24:13.543166+0000","last_clean":"2024-08-17T17:24:13.543166+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:24:13.543166+0000","last_undegraded":"2024-08-17T17:24:13.543166+0000","last_fullsized":"2024-08-17T17:24:13.543166+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":451,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.549655+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:24:13.549655+0000","last_peered":"2024-08-17T17:24:13.549655+0000","last_clean":"2024-08-17T17:24:13.549655+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:24:13.549655+0000","last_undegraded":"2024-08-17T17:24:13.549655+0000","last_fullsized":"2024-08-17T17:24:13.549655+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":157,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536645+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:24:13.536645+0000","last_peered":"2024-08-17T17:24:13.536645+0000","last_clean":"2024-08-17T17:24:13.536645+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:24:13.536645+0000","last_undegraded":"2024-08-17T17:24:13.536645+0000","last_fullsized":"2024-08-17T17:24:13.536645+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"155'13080","reported_seq":13227,"reported_epoch":155,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:20.655226+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:24:20.655226+0000","last_peered":"2024-08-17T17:24:20.655226+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:24:20.655226+0000","last_undegraded":"2024-08-17T17:24:20.655226+0000","last_fullsized":"2024-08-17T17:24:20.655226+0000","mapping_epoch":135,"log_start":"116'4888","ondisk_log_start":"116'4888","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209059840,"num_objects":3190,"num_object_clones":0,"num_object_copies":9570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3172,"num_objects_unfound":0,"num_objects_dirty":3190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13080,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":3190}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":154,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.549566+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:24:13.549566+0000","last_peered":"2024-08-17T17:24:13.549566+0000","last_clean":"2024-08-17T17:24:13.549566+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:24:13.549566+0000","last_undegraded":"2024-08-17T17:24:13.549566+0000","last_fullsized":"2024-08-17T17:24:13.549566+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":681,"reported_epoch":154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:23:57.457631+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:23:57.457631+0000","last_peered":"2024-08-17T17:23:57.457631+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:23:57.457631+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":152,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.524343+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:24:13.524343+0000","last_peered":"2024-08-17T17:24:13.524343+0000","last_clean":"2024-08-17T17:24:13.524343+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:24:13.524343+0000","last_undegraded":"2024-08-17T17:24:13.524343+0000","last_fullsized":"2024-08-17T17:24:13.524343+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"155'13160","reported_seq":13334,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.157373+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:24:16.157373+0000","last_peered":"2024-08-17T17:24:16.157373+0000","last_clean":"2024-08-17T17:24:16.157373+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:24:16.157373+0000","last_undegraded":"2024-08-17T17:24:16.157373+0000","last_fullsized":"2024-08-17T17:24:16.157373+0000","mapping_epoch":128,"log_start":"117'6475","ondisk_log_start":"117'6475","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218628096,"num_objects":3336,"num_object_clones":0,"num_object_copies":10008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13160,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":163,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.558363+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:24:13.558363+0000","last_peered":"2024-08-17T17:24:13.558363+0000","last_clean":"2024-08-17T17:24:13.558363+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:24:13.558363+0000","last_undegraded":"2024-08-17T17:24:13.558363+0000","last_fullsized":"2024-08-17T17:24:13.558363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":646,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:13.537207+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:24:13.537207+0000","last_peered":"2024-08-17T17:24:13.537207+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:24:13.537207+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.550795+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:24:13.550795+0000","last_peered":"2024-08-17T17:24:13.550795+0000","last_clean":"2024-08-17T17:24:13.550795+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:24:13.550795+0000","last_undegraded":"2024-08-17T17:24:13.550795+0000","last_fullsized":"2024-08-17T17:24:13.550795+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"155'13304","reported_seq":14621,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:18.522040+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:24:18.522040+0000","last_peered":"2024-08-17T17:24:18.522040+0000","last_clean":"2024-08-17T17:24:18.522040+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:24:18.522040+0000","last_undegraded":"2024-08-17T17:24:18.522040+0000","last_fullsized":"2024-08-17T17:24:18.522040+0000","mapping_epoch":139,"log_start":"117'5730","ondisk_log_start":"117'5730","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217841664,"num_objects":3324,"num_object_clones":0,"num_object_copies":9972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13304,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":182,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523071+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:24:13.523071+0000","last_peered":"2024-08-17T17:24:13.523071+0000","last_clean":"2024-08-17T17:24:13.523071+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:24:13.523071+0000","last_undegraded":"2024-08-17T17:24:13.523071+0000","last_fullsized":"2024-08-17T17:24:13.523071+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":398,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.497967+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:23:57.497967+0000","last_peered":"2024-08-17T17:23:57.497967+0000","last_clean":"2024-08-17T17:23:57.497967+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:23:57.497967+0000","last_undegraded":"2024-08-17T17:23:57.497967+0000","last_fullsized":"2024-08-17T17:23:57.497967+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459918+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:23:57.459918+0000","last_peered":"2024-08-17T17:23:57.459918+0000","last_clean":"2024-08-17T17:23:57.459918+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:23:57.459918+0000","last_undegraded":"2024-08-17T17:23:57.459918+0000","last_fullsized":"2024-08-17T17:23:57.459918+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460321+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:23:57.460321+0000","last_peered":"2024-08-17T17:23:57.460321+0000","last_clean":"2024-08-17T17:23:57.460321+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:23:57.460321+0000","last_undegraded":"2024-08-17T17:23:57.460321+0000","last_fullsized":"2024-08-17T17:23:57.460321+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":48,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460478+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:23:57.460478+0000","last_peered":"2024-08-17T17:23:57.460478+0000","last_clean":"2024-08-17T17:23:57.460478+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:23:57.460478+0000","last_undegraded":"2024-08-17T17:23:57.460478+0000","last_fullsized":"2024-08-17T17:23:57.460478+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":146,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536529+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:24:13.536529+0000","last_peered":"2024-08-17T17:24:13.536529+0000","last_clean":"2024-08-17T17:24:13.536529+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:24:13.536529+0000","last_undegraded":"2024-08-17T17:24:13.536529+0000","last_fullsized":"2024-08-17T17:24:13.536529+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":149,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.457419+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:23:57.457419+0000","last_peered":"2024-08-17T17:23:57.457419+0000","last_clean":"2024-08-17T17:23:57.457419+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:23:57.457419+0000","last_undegraded":"2024-08-17T17:23:57.457419+0000","last_fullsized":"2024-08-17T17:23:57.457419+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":83,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.458103+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:23:57.458103+0000","last_peered":"2024-08-17T17:23:57.458103+0000","last_clean":"2024-08-17T17:23:57.458103+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:23:57.458103+0000","last_undegraded":"2024-08-17T17:23:57.458103+0000","last_fullsized":"2024-08-17T17:23:57.458103+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.545935+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:24:13.545935+0000","last_peered":"2024-08-17T17:24:13.545935+0000","last_clean":"2024-08-17T17:24:13.545935+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:24:13.545935+0000","last_undegraded":"2024-08-17T17:24:13.545935+0000","last_fullsized":"2024-08-17T17:24:13.545935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":162,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.524050+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:24:13.524050+0000","last_peered":"2024-08-17T17:24:13.524050+0000","last_clean":"2024-08-17T17:24:13.524050+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:24:13.524050+0000","last_undegraded":"2024-08-17T17:24:13.524050+0000","last_fullsized":"2024-08-17T17:24:13.524050+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":164,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.548065+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:24:13.548065+0000","last_peered":"2024-08-17T17:24:13.548065+0000","last_clean":"2024-08-17T17:24:13.548065+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:24:13.548065+0000","last_undegraded":"2024-08-17T17:24:13.548065+0000","last_fullsized":"2024-08-17T17:24:13.548065+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536274+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:24:13.536274+0000","last_peered":"2024-08-17T17:24:13.536274+0000","last_clean":"2024-08-17T17:24:13.536274+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:24:13.536274+0000","last_undegraded":"2024-08-17T17:24:13.536274+0000","last_fullsized":"2024-08-17T17:24:13.536274+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":180,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.550732+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:24:13.550732+0000","last_peered":"2024-08-17T17:24:13.550732+0000","last_clean":"2024-08-17T17:24:13.550732+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:24:13.550732+0000","last_undegraded":"2024-08-17T17:24:13.550732+0000","last_fullsized":"2024-08-17T17:24:13.550732+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":156,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523265+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:24:13.523265+0000","last_peered":"2024-08-17T17:24:13.523265+0000","last_clean":"2024-08-17T17:24:13.523265+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:24:13.523265+0000","last_undegraded":"2024-08-17T17:24:13.523265+0000","last_fullsized":"2024-08-17T17:24:13.523265+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":160,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536420+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:24:13.536420+0000","last_peered":"2024-08-17T17:24:13.536420+0000","last_clean":"2024-08-17T17:24:13.536420+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:24:13.536420+0000","last_undegraded":"2024-08-17T17:24:13.536420+0000","last_fullsized":"2024-08-17T17:24:13.536420+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":177,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.542989+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:24:13.542989+0000","last_peered":"2024-08-17T17:24:13.542989+0000","last_clean":"2024-08-17T17:24:13.542989+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:24:13.542989+0000","last_undegraded":"2024-08-17T17:24:13.542989+0000","last_fullsized":"2024-08-17T17:24:13.542989+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459856+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:23:57.459856+0000","last_peered":"2024-08-17T17:23:57.459856+0000","last_clean":"2024-08-17T17:23:57.459856+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:23:57.459856+0000","last_undegraded":"2024-08-17T17:23:57.459856+0000","last_fullsized":"2024-08-17T17:23:57.459856+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.564124+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:24:13.564124+0000","last_peered":"2024-08-17T17:24:13.564124+0000","last_clean":"2024-08-17T17:24:13.564124+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:24:13.564124+0000","last_undegraded":"2024-08-17T17:24:13.564124+0000","last_fullsized":"2024-08-17T17:24:13.564124+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":162,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523435+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:24:13.523435+0000","last_peered":"2024-08-17T17:24:13.523435+0000","last_clean":"2024-08-17T17:24:13.523435+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:24:13.523435+0000","last_undegraded":"2024-08-17T17:24:13.523435+0000","last_fullsized":"2024-08-17T17:24:13.523435+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460182+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:23:57.460182+0000","last_peered":"2024-08-17T17:23:57.460182+0000","last_clean":"2024-08-17T17:23:57.460182+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:23:57.460182+0000","last_undegraded":"2024-08-17T17:23:57.460182+0000","last_fullsized":"2024-08-17T17:23:57.460182+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":139,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460409+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:23:57.460409+0000","last_peered":"2024-08-17T17:23:57.460409+0000","last_clean":"2024-08-17T17:23:57.460409+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:23:57.460409+0000","last_undegraded":"2024-08-17T17:23:57.460409+0000","last_fullsized":"2024-08-17T17:23:57.460409+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459773+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:23:57.459773+0000","last_peered":"2024-08-17T17:23:57.459773+0000","last_clean":"2024-08-17T17:23:57.459773+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:23:57.459773+0000","last_undegraded":"2024-08-17T17:23:57.459773+0000","last_fullsized":"2024-08-17T17:23:57.459773+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":124,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.460789+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:23:57.460789+0000","last_peered":"2024-08-17T17:23:57.460789+0000","last_clean":"2024-08-17T17:23:57.460789+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:23:57.460789+0000","last_undegraded":"2024-08-17T17:23:57.460789+0000","last_fullsized":"2024-08-17T17:23:57.460789+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":182,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.546178+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:24:13.546178+0000","last_peered":"2024-08-17T17:24:13.546178+0000","last_clean":"2024-08-17T17:24:13.546178+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:24:13.546178+0000","last_undegraded":"2024-08-17T17:24:13.546178+0000","last_fullsized":"2024-08-17T17:24:13.546178+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":170,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.543549+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:24:13.543549+0000","last_peered":"2024-08-17T17:24:13.543549+0000","last_clean":"2024-08-17T17:24:13.543549+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:24:13.543549+0000","last_undegraded":"2024-08-17T17:24:13.543549+0000","last_fullsized":"2024-08-17T17:24:13.543549+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":154,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.459502+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:23:57.459502+0000","last_peered":"2024-08-17T17:23:57.459502+0000","last_clean":"2024-08-17T17:23:57.459502+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:23:57.459502+0000","last_undegraded":"2024-08-17T17:23:57.459502+0000","last_fullsized":"2024-08-17T17:23:57.459502+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":173,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.523945+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:24:13.523945+0000","last_peered":"2024-08-17T17:24:13.523945+0000","last_clean":"2024-08-17T17:24:13.523945+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:24:13.523945+0000","last_undegraded":"2024-08-17T17:24:13.523945+0000","last_fullsized":"2024-08-17T17:24:13.523945+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":3480289304,"num_objects":53106,"num_object_clones":0,"num_object_copies":159318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35075,"num_objects_misplaced":10067,"num_objects_unfound":0,"num_objects_dirty":53106,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":210778,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17354,"num_bytes_recovered":971702272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11656245248,"data_stored":11656233032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131208,"ondisk_log_size":131208,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201529,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1204876,"kb_used_data":1087620,"kb_used_omap":1,"kb_used_meta":117246,"kb_avail":92540276,"statfs":{"total":95995035648,"available":94761242624,"internally_reserved":0,"allocated":1113722880,"data_stored":1112832427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":120060363},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463168,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918784,"kb_used_data":837624,"kb_used_omap":1,"kb_used_meta":81150,"kb_avail":92826368,"statfs":{"total":95995035648,"available":95054200832,"internally_reserved":0,"allocated":857726976,"data_stored":856848977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":83098055},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724804,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986439,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1941568,"kb_used_data":1761976,"kb_used_omap":1,"kb_used_meta":179582,"kb_avail":91803584,"statfs":{"total":95995035648,"available":94006870016,"internally_reserved":0,"allocated":1804263424,"data_stored":1803384629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":183892422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,6,6,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435331,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957936,"kb_used_data":858152,"kb_used_omap":5,"kb_used_meta":99706,"kb_avail":92787216,"statfs":{"total":95995035648,"available":95014109184,"internally_reserved":0,"allocated":878747648,"data_stored":877880507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":102099017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598857,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428404,"kb_used_data":1274540,"kb_used_omap":7,"kb_used_meta":153848,"kb_avail":92316748,"statfs":{"total":95995035648,"available":94532349952,"internally_reserved":0,"allocated":1305128960,"data_stored":1304258203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":157540525},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762384,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161704,"kb_used_data":1057760,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":92583448,"statfs":{"total":95995035648,"available":94805450752,"internally_reserved":0,"allocated":1083146240,"data_stored":1082286877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":106357153},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925910,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2099448,"kb_used_data":1901104,"kb_used_omap":7,"kb_used_meta":198264,"kb_avail":91645704,"statfs":{"total":95995035648,"available":93845200896,"internally_reserved":0,"allocated":1946730496,"data_stored":1945856424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":203022940},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1939603456,"data_stored":1939603456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1065680896,"data_stored":1065680896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1296437248,"data_stored":1296433176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":868352000,"data_stored":868352000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1794834432,"data_stored":1794834432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2745896960,"data_stored":2745892888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":844562432,"data_stored":844562432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1100877824,"data_stored":1100873752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:25.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:24:25.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:25.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.704 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:25.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:25.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.979 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:26.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:26.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:26.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:26.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:26.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:26.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:26 smithi064 ceph-mon[104516]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 35209/150156 objects degraded (23.448%); 9610/150156 objects misplaced (6.400%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:26.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:26 smithi064 ceph-mon[109246]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 35209/150156 objects degraded (23.448%); 9610/150156 objects misplaced (6.400%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:26.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:26.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:26.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:26.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:26.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:26 smithi175 ceph-mon[101434]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 35209/150156 objects degraded (23.448%); 9610/150156 objects misplaced (6.400%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:24:26.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:26.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:26.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:26.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:26.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:26.978 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:27.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:27.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:27.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:27.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:27.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:27.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:27.382 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:27.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:27.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:27.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:27.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:27.756 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:27.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:27.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:27.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:27.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:28.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:28.058 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:28.100 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:24:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:24:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:96003 5.c 5:306044c8:::benchmark_data_smithi064_147470_object96002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:26.735804+0000", 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 179.71945040099999, 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10317612299999999, 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96003 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735804+0000", 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735804+0000", 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735806+0000", 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.984e-06 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735830+0000", 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4462999999999999e-05 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735848+0000", 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8008000000000001e-05 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735857+0000", 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8619999999999992e-06 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.735928+0000", 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1358000000000004e-05 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736104+0000", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175377 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.736267+0000", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000163053 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.740209+0000", 2024-08-17T17:24:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039420530000000001 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.740635+0000", 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042590299999999999 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.838866+0000", 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098231365000000001 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.838929+0000", 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2447999999999996e-05 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:26.838980+0000", 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1248999999999997e-05 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:124739 5.c 5:366c381d:::benchmark_data_smithi064_147470_object124738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-08-17T17:24:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:54.185030+0000", 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.27022362599999, 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094986084999999998, 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124739 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185030+0000", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185030+0000", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185032+0000", 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.517e-06 2024-08-17T17:24:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185056+0000", 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4165e-05 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185068+0000", 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2303999999999999e-05 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.185076+0000", 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.841e-06 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.188401+0000", 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033251489999999999 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.188523+0000", 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121087 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.188651+0000", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012892200000000001 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.196189+0000", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007537897 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.197235+0000", 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001045741 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.279699+0000", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082464019 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.279861+0000", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016165099999999999 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.280017+0000", 2024-08-17T17:24:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155792 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:124743 5.c 5:33d09864:::benchmark_data_smithi064_147470_object124742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:21:54.187500+0000", 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.26775405500001, 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.092976011999999997, 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124743 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187500+0000", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187500+0000", 2024-08-17T17:24:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187502+0000", 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.601e-06 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187523+0000", 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1094000000000002e-05 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187533+0000", 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0023999999999999e-05 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.187539+0000", 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3799999999999999e-06 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.192263+0000", 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047237010000000003 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.192350+0000", 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6854000000000006e-05 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.192434+0000", 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3949000000000005e-05 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.196271+0000", 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038375750000000002 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.197353+0000", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001082235 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.280183+0000", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082829765999999999 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.280353+0000", 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017006500000000001 2024-08-17T17:24:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:21:54.280476+0000", 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012276799999999999 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131915 5.c 5:365a0339:::benchmark_data_smithi064_147470_object131914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 144.42364460900001, 2024-08-17T17:24:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1561467159999999, 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131915 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:24:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031611+0000", 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6780000000000001e-06 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031633+0000", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1382e-05 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031647+0000", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4217e-05 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031653+0000", 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8119999999999996e-06 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031715+0000", 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2317000000000007e-05 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031889+0000", 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173826 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.032068+0000", 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017936499999999999 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050025+0000", 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017957101 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050038+0000", 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3128e-05 2024-08-17T17:24:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049267+0000", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.999229005 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049283+0000", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5880999999999999e-05 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.187675+0000", 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.138391844 2024-08-17T17:24:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.187756+0000", 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1160000000000002e-05 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131927 5.c 5:3e0315ab:::benchmark_data_smithi064_147470_object131926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 144.39110045800001, 2024-08-17T17:24:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1700607590000001, 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131927 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064156+0000", 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4099999999999998e-06 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064196+0000", 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9903000000000001e-05 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064219+0000", 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2807e-05 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064229+0000", 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0387999999999999e-05 2024-08-17T17:24:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064439+0000", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021010700000000001 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064462+0000", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2501999999999999e-05 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049376+0000", 2024-08-17T17:24:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98491408300000005 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049385+0000", 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.188213+0000", 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1388277579999999 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.188382+0000", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016944599999999999 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.189048+0000", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066579400000000004 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203168+0000", 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014119567 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.207367+0000", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041998019999999999 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.218319+0000", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010952147000000001 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234010+0000", 2024-08-17T17:24:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015690763999999999 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234083+0000", 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3046999999999994e-05 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234214+0000", 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013113400000000001 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131930 5.c 5:32de370a:::benchmark_data_smithi064_147470_object131929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 144.387757486, 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1672973080000002, 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131930 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067498+0000", 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.747e-06 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067526+0000", 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7480000000000001e-05 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067544+0000", 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8377999999999999e-05 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067553+0000", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8380000000000008e-06 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067608+0000", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4829e-05 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067625+0000", 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6599000000000002e-05 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049418+0000", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98179379899999997 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049428+0000", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0128e-05 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203205+0000", 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1537763000000001 2024-08-17T17:24:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203366+0000", 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016128200000000001 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203557+0000", 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019087600000000001 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.207352+0000", 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037946690000000001 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.215942+0000", 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085902219999999998 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.218428+0000", 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024865429999999999 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234619+0000", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016191082999999998 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234679+0000", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9117999999999999e-05 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234794+0000", 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115417 2024-08-17T17:24:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140817 5.c 5:3beb597f:::benchmark_data_smithi064_147470_object8873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.365733+0000", 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 125.08952150499999, 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85906704499999997, 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140817 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365733+0000", 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365733+0000", 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365735+0000", 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1399999999999998e-06 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365743+0000", 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3489999999999992e-06 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365764+0000", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1112999999999999e-05 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365774+0000", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8160000000000005e-06 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365846+0000", 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1923999999999998e-05 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.365856+0000", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7350000000000008e-06 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.210937+0000", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84508150100000001 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.211058+0000", 2024-08-17T17:24:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012099600000000001 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.211202+0000", 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143623 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.217794+0000", 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065920919999999999 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.217844+0000", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0044999999999999e-05 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224676+0000", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068317940000000004 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224757+0000", 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1067000000000002e-05 2024-08-17T17:24:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224800+0000", 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2849999999999998e-05 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140823 5.c 5:3f8a539c:::benchmark_data_smithi064_147470_object8879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.377212+0000", 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 125.078042564, 2024-08-17T17:24:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85248439600000003, 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140823 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377212+0000", 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377212+0000", 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377214+0000", 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0150000000000002e-06 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377220+0000", 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9499999999999998e-06 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377239+0000", 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9802000000000001e-05 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377249+0000", 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2220000000000008e-06 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377336+0000", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7077000000000007e-05 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377345+0000", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2359999999999993e-06 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.214876+0000", 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83753072699999997 2024-08-17T17:24:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.215017+0000", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141757 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.215148+0000", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013088600000000001 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224592+0000", 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094440749999999997 2024-08-17T17:24:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.228898+0000", 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043057929999999996 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229583+0000", 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068521599999999999 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229633+0000", 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9537999999999999e-05 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229696+0000", 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3102000000000007e-05 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140825 5.c 5:37e61b3f:::benchmark_data_smithi064_147470_object8881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:24:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.377394+0000", 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 125.07785990000001, 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85238849800000005, 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140825 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377394+0000", 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377394+0000", 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377395+0000", 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4300000000000003e-07 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377398+0000", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5260000000000002e-06 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377410+0000", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1928e-05 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377417+0000", 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6529999999999997e-06 2024-08-17T17:24:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377483+0000", 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5887000000000001e-05 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.377490+0000", 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4290000000000004e-06 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.217876+0000", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84038526199999997 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.217998+0000", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012228299999999999 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.218099+0000", 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010111300000000001 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224659+0000", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065603110000000001 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229058+0000", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043983249999999998 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229704+0000", 2024-08-17T17:24:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064603299999999996 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229744+0000", 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0326e-05 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229783+0000", 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8779000000000003e-05 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140859 5.c 5:3db2c3ee:::benchmark_data_smithi064_147470_object8915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.416081+0000", 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 125.039172861, 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81377440899999998, 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140859 2024-08-17T17:24:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416081+0000", 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416081+0000", 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416083+0000", 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5149999999999999e-06 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416087+0000", 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4159999999999997e-06 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416099+0000", 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.203e-05 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416105+0000", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0789999999999997e-06 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416178+0000", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2908000000000001e-05 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.416185+0000", 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.708e-06 2024-08-17T17:24:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.221299+0000", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80511434900000001 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.221464+0000", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164341 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.221585+0000", 2024-08-17T17:24:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012161 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.228418+0000", 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068325299999999999 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229094+0000", 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067628499999999999 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229790+0000", 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069613800000000001 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229823+0000", 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2703000000000002e-05 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229856+0000", 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2796999999999998e-05 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140872 5.c 5:36a4a86f:::benchmark_data_smithi064_147470_object8928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:21.440200+0000", 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 125.015053915, 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.794865186, 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140872 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440200+0000", 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440200+0000", 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440202+0000", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7430000000000001e-06 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440207+0000", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8099999999999997e-06 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440221+0000", 2024-08-17T17:24:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3974e-05 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440228+0000", 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6499999999999996e-06 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440285+0000", 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6815999999999999e-05 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.440292+0000", 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.37e-06 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224830+0000", 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78453817299999995 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.224950+0000", 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120491 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.225054+0000", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010358500000000001 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.229863+0000", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048092689999999997 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.230143+0000", 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027979900000000001 2024-08-17T17:24:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.234964+0000", 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004821077 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.235020+0000", 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6137000000000001e-05 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:22.235065+0000", 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5291999999999999e-05 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:149990 5.7 5:e6ca93a2:::benchmark_data_smithi064_147470_object18046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:36.563257+0000", 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 109.89199675, 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098480867999999999, 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149990 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563257+0000", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563257+0000", 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563260+0000", 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.243e-06 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563267+0000", 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3159999999999999e-06 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563286+0000", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9283000000000002e-05 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.563295+0000", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2730000000000008e-06 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.568021+0000", 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047256980000000004 2024-08-17T17:24:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.568179+0000", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158138 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.568340+0000", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016116500000000001 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.571700+0000", 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033590909999999998 2024-08-17T17:24:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.579664+0000", 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079645080000000004 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.657982+0000", 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.078317506999999995 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.658237+0000", 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025581600000000002 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:36.661738+0000", 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035008299999999999 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:151017 5.7 5:ed9c901c:::benchmark_data_smithi064_147470_object19073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-17T17:24:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:38.280667+0000", 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.17458666, 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.102428831, 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151017 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280667+0000", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280667+0000", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280668+0000", 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5600000000000005e-07 2024-08-17T17:24:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280672+0000", 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8550000000000004e-06 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280710+0000", 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8056999999999998e-05 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.280718+0000", 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4359999999999996e-06 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.289919+0000", 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009201898 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.290315+0000", 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039505399999999999 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.290789+0000", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047438999999999999 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.295524+0000", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047347149999999996 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.299793+0000", 2024-08-17T17:24:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042698570000000002 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.379624+0000", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079830130999999999 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.379693+0000", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9107000000000001e-05 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:38.383096+0000", 2024-08-17T17:24:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034035749999999998 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:152801 5.7 5:e51e6066:::benchmark_data_smithi064_147470_object20857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:41.951168+0000", 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.504085741, 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.130683837, 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152801 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951168+0000", 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951168+0000", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951170+0000", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.483e-06 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951175+0000", 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7729999999999999e-06 2024-08-17T17:24:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951189+0000", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.446e-05 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.951196+0000", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9709999999999998e-06 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.957233+0000", 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060369380000000004 2024-08-17T17:24:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.957380+0000", 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000147393 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.957528+0000", 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014801699999999999 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.961177+0000", 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036482139999999999 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:41.962314+0000", 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011378409999999999 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:42.078306+0000", 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.115991829 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:42.078372+0000", 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5791999999999996e-05 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:42.081852+0000", 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034801260000000001 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:163138 5.7 5:eb838d80:::benchmark_data_smithi064_147470_object31194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:59.685406+0000", 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 86.769847846000005, 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103145854, 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 163138 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685406+0000", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685406+0000", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685409+0000", 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.294e-06 2024-08-17T17:24:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685414+0000", 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9159999999999998e-06 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685426+0000", 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1595e-05 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.685433+0000", 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0870000000000001e-06 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.687870+0000", 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024363850000000001 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.687984+0000", 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114607 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.688091+0000", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010706599999999999 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.690615+0000", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025234860000000001 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.696980+0000", 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063654139999999998 2024-08-17T17:24:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.784735+0000", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.087754701000000004 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.784836+0000", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100937 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:59.788552+0000", 2024-08-17T17:24:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003716366 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:167113 5.7 5:eed3c007:::benchmark_data_smithi064_147470_object35169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:06.387878+0000", 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.067375771000002, 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103540249, 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 167113 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387878+0000", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387878+0000", 2024-08-17T17:24:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387879+0000", 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2699999999999998e-07 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387893+0000", 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4036000000000001e-05 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387907+0000", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3491e-05 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.387914+0000", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2989999999999999e-06 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.392210+0000", 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042957840000000004 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.392645+0000", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000435599 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.393084+0000", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000438539 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.403598+0000", 2024-08-17T17:24:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010514177 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.403738+0000", 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013999600000000001 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.487390+0000", 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.083651666999999999 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.487662+0000", 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027229100000000003 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.491419+0000", 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037564429999999999 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:167117 5.7 5:ed31ee7f:::benchmark_data_smithi064_147470_object35173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-17T17:24:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:06.391264+0000", 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.063989831000001, 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103315268, 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 167117 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391264+0000", 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391264+0000", 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391266+0000", 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6419999999999999e-06 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391272+0000", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7819999999999999e-06 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391288+0000", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6589999999999999e-05 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.391297+0000", 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4570000000000005e-06 2024-08-17T17:24:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.399105+0000", 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078085669999999998 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.399826+0000", 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000720617 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400097+0000", 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027141099999999999 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.403765+0000", 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036675060000000001 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.418178+0000", 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014413173 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.491482+0000", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073303679999999996 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.491544+0000", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2718999999999994e-05 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.494580+0000", 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030351240000000002 2024-08-17T17:24:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:167123 5.7 5:ea485c5e:::benchmark_data_smithi064_147470_object35179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:06.400753+0000", 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.054501028999994, 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10161395099999999, 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 167123 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400753+0000", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400753+0000", 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400754+0000", 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1370000000000001e-06 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400758+0000", 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5650000000000002e-06 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400767+0000", 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6139999999999998e-06 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.400774+0000", 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6209999999999998e-06 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.403801+0000", 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030273269999999998 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.403978+0000", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000176878 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.404155+0000", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000176508 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.418128+0000", 2024-08-17T17:24:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013973604000000001 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.418241+0000", 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011246700000000001 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.498834+0000", 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.080593078999999998 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.498950+0000", 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011602799999999999 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:06.502367+0000", 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034171230000000002 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:197861 5.7 5:eeb929fc:::benchmark_data_smithi064_147470_object65917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:57.119788+0000", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.335466124, 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098608534999999997, 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 197861 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119788+0000", 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119788+0000", 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119790+0000", 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5319999999999999e-06 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119794+0000", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.758e-06 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119827+0000", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2650000000000001e-05 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.119836+0000", 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6260000000000001e-06 2024-08-17T17:24:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.124163+0000", 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043269559999999999 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.124471+0000", 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030896500000000002 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.124922+0000", 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045005799999999998 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.146377+0000", 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021455084999999999 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.152212+0000", 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058353959999999996 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.215026+0000", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062814307999999999 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.215084+0000", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7766000000000002e-05 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.218397+0000", 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033124349999999999 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:197867 5.7 5:e8a3c506:::benchmark_data_smithi064_147470_object65923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:57.128074+0000", 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.327179684000001, 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093297943999999994, 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 197867 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128074+0000", 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128074+0000", 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128076+0000", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.84e-06 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128082+0000", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1840000000000004e-06 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128099+0000", 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6852999999999999e-05 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.128108+0000", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3869999999999995e-06 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.132054+0000", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039463220000000004 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.132759+0000", 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070511399999999996 2024-08-17T17:24:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.139068+0000", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006309098 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.152260+0000", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013191588000000001 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.152289+0000", 2024-08-17T17:24:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9634000000000001e-05 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.218456+0000", 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.066166313000000004 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.218520+0000", 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4330000000000002e-05 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:28.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:57.221372+0000", 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028522809999999999 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:28.239 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:24:28.239 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:28.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:28.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:28.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:28 smithi064 ceph-mon[104516]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35323/146721 objects degraded (24.075%); 9065/146721 objects misplaced (6.178%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:24:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35323/146721 objects degraded (24.075%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:28 smithi064 ceph-mon[109246]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35323/146721 objects degraded (24.075%); 9065/146721 objects misplaced (6.178%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:24:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35323/146721 objects degraded (24.075%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:28.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:28.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:28.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:28 smithi175 ceph-mon[101434]: pgmap v459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35323/146721 objects degraded (24.075%); 9065/146721 objects misplaced (6.178%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:24:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35323/146721 objects degraded (24.075%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:28.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:28.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:28.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:29.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:29.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:29.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:29.240 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:29.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:29.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:29.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:24:29.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:29.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:29.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:29.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:29.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:29.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:29.738 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:24:29.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:29.775 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:24:29.775 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:29.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:24:29.820 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:29.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:29.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:24:29.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:29.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:29.897 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:24:29.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:29.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:24:29.941 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:29.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:24:29.986 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:29.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:30.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:24:30.028 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:30.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:30.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:30.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:30.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:30.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:30.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:30 smithi064 ceph-mon[104516]: pgmap v460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 35323/140661 objects degraded (25.112%); 9065/140661 objects misplaced (6.445%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:30.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/509180408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:30 smithi064 ceph-mon[109246]: pgmap v460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 35323/140661 objects degraded (25.112%); 9065/140661 objects misplaced (6.445%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:30.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/509180408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:30.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:30.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:30.601 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:30 smithi175 ceph-mon[101434]: pgmap v460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 35323/140661 objects degraded (25.112%); 9065/140661 objects misplaced (6.445%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/509180408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:30.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:30.728 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:30.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:30.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:30.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:30.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:30.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:31.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:31.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:31.158 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:31.205 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:31.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:31.247 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:24:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:31.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:31.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:31.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2238596888' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:31 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2238596888' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:31 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.450 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:31.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:31.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2238596888' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:31 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:31.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:31.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:31.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:31.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:31.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:32.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:32.108 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:32.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:32.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:32.262 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:32.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:32.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[104516]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 35323/140160 objects degraded (25.202%); 8899/140160 objects misplaced (6.349%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[104516]: osdmap e156: 8 total, 8 up, 7 in 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[109246]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 35323/140160 objects degraded (25.202%); 8899/140160 objects misplaced (6.349%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:32 smithi064 ceph-mon[109246]: osdmap e156: 8 total, 8 up, 7 in 2024-08-17T17:24:32.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:32.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:32.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:32.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:32 smithi175 ceph-mon[101434]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 35323/140160 objects degraded (25.202%); 8899/140160 objects misplaced (6.349%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:24:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:32 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:32 smithi175 ceph-mon[101434]: osdmap e156: 8 total, 8 up, 7 in 2024-08-17T17:24:32.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:32.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:32.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:32.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:32.930 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:33.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:33.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:33.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:33.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:33.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:33.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:33.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:33.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:33.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35323/140160 objects degraded (25.202%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35323/140160 objects degraded (25.202%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:33.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35323/140160 objects degraded (25.202%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:33.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:33.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:33.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:33.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:33.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:34.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:34.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:34.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:24:34.171 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:33] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:24:34.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:34.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:34.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:34.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:34.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:34.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:34 smithi064 ceph-mon[104516]: pgmap v463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35425/134463 objects degraded (26.346%); 8502/134463 objects misplaced (6.323%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:34.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:34 smithi064 ceph-mon[109246]: pgmap v463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35425/134463 objects degraded (26.346%); 8502/134463 objects misplaced (6.323%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:34.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:34.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:34 smithi175 ceph-mon[101434]: pgmap v463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35425/134463 objects degraded (26.346%); 8502/134463 objects misplaced (6.323%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:34.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:34.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:34.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:34.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:34.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.020 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:35.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:35.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:35.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:35.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:35.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:35.353 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:35.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:35.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:35.524 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:35.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:35.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:35.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:35.676 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:24:35.676 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35425/131814 objects degraded (26.875%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":659,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":156,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915134,"num_in_osds":7,"osd_in_since":1723915354,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":78},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":43938,"data_bytes":2882410493,"bytes_used":8502472704,"bytes_avail":663462776832,"bytes_total":671965249536,"degraded_objects":35425,"degraded_total":131814,"degraded_ratio":0.26874990516940539,"misplaced_objects":8502,"misplaced_total":131814,"misplaced_ratio":0.064499977240657286,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1310112,"recovering_keys_per_sec":0,"num_objects_recovered":200,"num_bytes_recovered":13107200,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":611},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":23,"modified":"2024-08-17T17:19:54.695374+0000","services":{}},"progress_events":{}} 2024-08-17T17:24:35.714 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:35.714 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:35.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:35.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:35.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:35.935 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:36.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:36.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:36.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:36.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:36.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:36.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:36.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:36.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:36.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:36.617 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:36 smithi175 ceph-mon[101434]: pgmap v464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 35425/131814 objects degraded (26.875%); 8502/131814 objects misplaced (6.450%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3384342538' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:24:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:36 smithi064 ceph-mon[104516]: pgmap v464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 35425/131814 objects degraded (26.875%); 8502/131814 objects misplaced (6.450%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3384342538' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:24:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:36 smithi064 ceph-mon[109246]: pgmap v464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.7 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 35425/131814 objects degraded (26.875%); 8502/131814 objects misplaced (6.450%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3384342538' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:24:36.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:36.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:36.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:36.887 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:36.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:37.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:37.132 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:37.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:37.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:24:37.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.285 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:37.387 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:37.518 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:37.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.682 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:37.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.818 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:37.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:37.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:38.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:38.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:38.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:38.354 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:38.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:38 smithi064 ceph-mon[104516]: pgmap v465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 35515/128133 objects degraded (27.717%); 7884/128133 objects misplaced (6.153%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35515/128133 objects degraded (27.717%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:38 smithi064 ceph-mon[109246]: pgmap v465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 35515/128133 objects degraded (27.717%); 7884/128133 objects misplaced (6.153%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35515/128133 objects degraded (27.717%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:38.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.492 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:38.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.515 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:38.516 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:24:38.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:38.522 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:38 smithi175 ceph-mon[101434]: pgmap v465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 35515/128133 objects degraded (27.717%); 7884/128133 objects misplaced (6.153%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:38.522 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35515/128133 objects degraded (27.717%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:38.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:38.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:38.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:38.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:38.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:38.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:38.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:39.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:39.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:39.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:39.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:39.422 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:39.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:39.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:39.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:24:39.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:24:39.619 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:39.619 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:39.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:24:39.631 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:39.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:39.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:24:39.671 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:39.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:24:39.706 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:39.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:39.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:39.744 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:24:39.744 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:39.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:24:39.787 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:39.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:39.822 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:24:39.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:39.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:39.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:39.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:40.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:40.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:40.217 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:40.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:40.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:40.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[104516]: pgmap v466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 35515/122187 objects degraded (29.066%); 7884/122187 objects misplaced (6.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:40.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4130079538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:40.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[109246]: pgmap v466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 35515/122187 objects degraded (29.066%); 7884/122187 objects misplaced (6.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:40.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4130079538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:40.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.528 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:40.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:40.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:40.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:40 smithi175 ceph-mon[101434]: pgmap v466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 35515/122187 objects degraded (29.066%); 7884/122187 objects misplaced (6.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4130079538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:40.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:40.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:40.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:40.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:40.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:40.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:41.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:41.103 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:41.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:41.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:41.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:41.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:41.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:41.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:41.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:41.644 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:41.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:41.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:41.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:41.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:41.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:41.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:41.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:41.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:42.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:42.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:42.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:42.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:42.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:42 smithi064 ceph-mon[109246]: pgmap v467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 35515/121635 objects degraded (29.198%); 7700/121635 objects misplaced (6.330%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:42.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:42 smithi064 ceph-mon[104516]: pgmap v467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 35515/121635 objects degraded (29.198%); 7700/121635 objects misplaced (6.330%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:42.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:42.529 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:42.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:42.659 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:42.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:42 smithi175 ceph-mon[101434]: pgmap v467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 35515/121635 objects degraded (29.198%); 7700/121635 objects misplaced (6.330%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:24:42.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.799 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:42.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:42.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:42.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:42.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:43.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:43.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:43.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.231 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:43.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:43.265 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:24:43.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:43.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:43.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3617339582' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35515/121635 objects degraded (29.198%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3617339582' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35515/121635 objects degraded (29.198%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:43.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:43.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:43.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:43 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3617339582' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:43 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35515/121635 objects degraded (29.198%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:43.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:43.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:43.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:43.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:43] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:24:44.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:44.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:44.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:44.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.564 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:44 smithi175 ceph-mon[101434]: pgmap v468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 35584/115617 objects degraded (30.777%); 7275/115617 objects misplaced (6.292%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:44 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:44 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:44 smithi175 ceph-mon[101434]: osdmap e157: 8 total, 8 up, 7 in 2024-08-17T17:24:44.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[104516]: pgmap v468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 35584/115617 objects degraded (30.777%); 7275/115617 objects misplaced (6.292%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[104516]: osdmap e157: 8 total, 8 up, 7 in 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[109246]: pgmap v468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 35584/115617 objects degraded (30.777%); 7275/115617 objects misplaced (6.292%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:44 smithi064 ceph-mon[109246]: osdmap e157: 8 total, 8 up, 7 in 2024-08-17T17:24:44.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:44.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:44.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:44.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:45.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:45.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:45.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:45.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:45.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:45.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.691 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.793 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:46.040 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:24:46.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:24:46.040 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:24:46.040 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:24:46.040 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:24:46.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:46.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:46.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:46.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:46.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:46.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:46.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:46 smithi064 ceph-mon[104516]: pgmap v470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 35584/112635 objects degraded (31.592%); 7275/112635 objects misplaced (6.459%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:24:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:46 smithi064 ceph-mon[109246]: pgmap v470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 35584/112635 objects degraded (31.592%); 7275/112635 objects misplaced (6.459%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:24:46.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:46.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:46.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:46 smithi175 ceph-mon[101434]: pgmap v470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 35584/112635 objects degraded (31.592%); 7275/112635 objects misplaced (6.459%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:24:46.685 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:46.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:46.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:46.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:46.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:46.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:47.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:24:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.136 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:47.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:47.264 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:24:47.266 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:47.269 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:24:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:47.276 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:47.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.440 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:47.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:47.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:47.701 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:47.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:47.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:24:47.831 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:47.857 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:47.959 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:47.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:48.113 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.155 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":470,"stamp":"2024-08-17T17:24:44.849451+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2463438845,"num_objects":37545,"num_object_clones":0,"num_object_copies":112635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35584,"num_objects_misplaced":7275,"num_objects_unfound":0,"num_objects_dirty":37545,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":227490,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17959,"num_bytes_recovered":1012669604,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132279,"ondisk_log_size":132279,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7263060,"kb_used_data":6242200,"kb_used_omap":33,"kb_used_meta":1020702,"kb_avail":648953004,"statfs":{"total":671965249536,"available":664527876096,"internally_reserved":0,"allocated":6392012800,"data_stored":6385831356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1045199435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,1,8,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-418971648,"num_objects":-6393,"num_object_clones":0,"num_object_copies":-19179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":159,"num_objects_misplaced":-1227,"num_objects_unfound":0,"num_objects_dirty":-6393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6393,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006947"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.284792+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:24:43.284792+0000","last_peered":"2024-08-17T17:24:43.284792+0000","last_clean":"2024-08-17T17:24:43.284792+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:24:43.284792+0000","last_undegraded":"2024-08-17T17:24:43.284792+0000","last_fullsized":"2024-08-17T17:24:43.284792+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":150,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.869627+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:24:15.869627+0000","last_peered":"2024-08-17T17:24:15.869627+0000","last_clean":"2024-08-17T17:24:15.869627+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:24:15.869627+0000","last_undegraded":"2024-08-17T17:24:15.869627+0000","last_fullsized":"2024-08-17T17:24:15.869627+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551172+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:24:13.551172+0000","last_peered":"2024-08-17T17:24:13.551172+0000","last_clean":"2024-08-17T17:24:13.551172+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:24:13.551172+0000","last_undegraded":"2024-08-17T17:24:13.551172+0000","last_fullsized":"2024-08-17T17:24:13.551172+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":158,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.261014+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:24:31.261014+0000","last_peered":"2024-08-17T17:24:31.261014+0000","last_clean":"2024-08-17T17:24:31.261014+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:24:31.261014+0000","last_undegraded":"2024-08-17T17:24:31.261014+0000","last_fullsized":"2024-08-17T17:24:31.261014+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":158,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.269992+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:24:31.269992+0000","last_peered":"2024-08-17T17:24:31.269992+0000","last_clean":"2024-08-17T17:24:31.269992+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:24:31.269992+0000","last_undegraded":"2024-08-17T17:24:31.269992+0000","last_fullsized":"2024-08-17T17:24:31.269992+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.869607+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:24:15.869607+0000","last_peered":"2024-08-17T17:24:15.869607+0000","last_clean":"2024-08-17T17:24:15.869607+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:24:15.869607+0000","last_undegraded":"2024-08-17T17:24:15.869607+0000","last_fullsized":"2024-08-17T17:24:15.869607+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":168,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.284810+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:24:43.284810+0000","last_peered":"2024-08-17T17:24:43.284810+0000","last_clean":"2024-08-17T17:24:43.284810+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:24:43.284810+0000","last_undegraded":"2024-08-17T17:24:43.284810+0000","last_fullsized":"2024-08-17T17:24:43.284810+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":145,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.716159+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:24:15.716159+0000","last_peered":"2024-08-17T17:24:15.716159+0000","last_clean":"2024-08-17T17:24:15.716159+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:24:15.716159+0000","last_undegraded":"2024-08-17T17:24:15.716159+0000","last_fullsized":"2024-08-17T17:24:15.716159+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"156'14033","reported_seq":15205,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:40.879707+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:24:40.879707+0000","last_peered":"2024-08-17T17:24:40.879707+0000","last_clean":"2024-08-17T17:24:40.879707+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:24:40.879707+0000","last_undegraded":"2024-08-17T17:24:40.879707+0000","last_fullsized":"2024-08-17T17:24:40.879707+0000","mapping_epoch":135,"log_start":"117'5900","ondisk_log_start":"117'5900","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8133,"log_dups_size":0,"ondisk_log_size":8133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14033,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":145,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551689+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:24:13.551689+0000","last_peered":"2024-08-17T17:24:13.551689+0000","last_clean":"2024-08-17T17:24:13.551689+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:24:13.551689+0000","last_undegraded":"2024-08-17T17:24:13.551689+0000","last_fullsized":"2024-08-17T17:24:13.551689+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":33,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.550869+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:24:13.550869+0000","last_peered":"2024-08-17T17:24:13.550869+0000","last_clean":"2024-08-17T17:24:13.550869+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:24:13.550869+0000","last_undegraded":"2024-08-17T17:24:13.550869+0000","last_fullsized":"2024-08-17T17:24:13.550869+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"156'14253","reported_seq":14355,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:42.606138+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:24:42.606138+0000","last_peered":"2024-08-17T17:24:42.606138+0000","last_clean":"2024-08-17T17:24:42.606138+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:24:42.606138+0000","last_undegraded":"2024-08-17T17:24:42.606138+0000","last_fullsized":"2024-08-17T17:24:42.606138+0000","mapping_epoch":111,"log_start":"116'4200","ondisk_log_start":"116'4200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10053,"log_dups_size":0,"ondisk_log_size":10053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147390464,"num_objects":2249,"num_object_clones":0,"num_object_copies":6747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14253,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":146,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551662+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:24:13.551662+0000","last_peered":"2024-08-17T17:24:13.551662+0000","last_clean":"2024-08-17T17:24:13.551662+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:24:13.551662+0000","last_undegraded":"2024-08-17T17:24:13.551662+0000","last_fullsized":"2024-08-17T17:24:13.551662+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.243410+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:24:31.243410+0000","last_peered":"2024-08-17T17:24:31.243410+0000","last_clean":"2024-08-17T17:24:31.243410+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:24:31.243410+0000","last_undegraded":"2024-08-17T17:24:31.243410+0000","last_fullsized":"2024-08-17T17:24:31.243410+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"156'14039","reported_seq":14146,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:42.592350+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:24:42.592350+0000","last_peered":"2024-08-17T17:24:42.592350+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:24:42.592350+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"115'4000","ondisk_log_start":"115'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14039,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-4715},{"shards":"0,2,5","objects":2263}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":117,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551299+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:24:13.551299+0000","last_peered":"2024-08-17T17:24:13.551299+0000","last_clean":"2024-08-17T17:24:13.551299+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:24:13.551299+0000","last_undegraded":"2024-08-17T17:24:13.551299+0000","last_fullsized":"2024-08-17T17:24:13.551299+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.535624+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:24:13.535624+0000","last_peered":"2024-08-17T17:24:13.535624+0000","last_clean":"2024-08-17T17:24:13.535624+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:24:13.535624+0000","last_undegraded":"2024-08-17T17:24:13.535624+0000","last_fullsized":"2024-08-17T17:24:13.535624+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"156'14125","reported_seq":15375,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:42.651375+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:24:42.651375+0000","last_peered":"2024-08-17T17:24:42.651375+0000","last_clean":"2024-08-17T17:24:42.651375+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:24:42.651375+0000","last_undegraded":"2024-08-17T17:24:42.651375+0000","last_fullsized":"2024-08-17T17:24:42.651375+0000","mapping_epoch":130,"log_start":"116'4100","ondisk_log_start":"116'4100","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153944064,"num_objects":2349,"num_object_clones":0,"num_object_copies":7047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14125,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.284938+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:24:43.284938+0000","last_peered":"2024-08-17T17:24:43.284938+0000","last_clean":"2024-08-17T17:24:43.284938+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:24:43.284938+0000","last_undegraded":"2024-08-17T17:24:43.284938+0000","last_fullsized":"2024-08-17T17:24:43.284938+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":172,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.261553+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:24:31.261553+0000","last_peered":"2024-08-17T17:24:31.261553+0000","last_clean":"2024-08-17T17:24:31.261553+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:24:31.261553+0000","last_undegraded":"2024-08-17T17:24:31.261553+0000","last_fullsized":"2024-08-17T17:24:31.261553+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"156'14184","reported_seq":14291,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:42.639412+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:24:42.639412+0000","last_peered":"2024-08-17T17:24:42.639412+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:24:42.639412+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"115'4100","ondisk_log_start":"115'4100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154337304,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14184,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":2356},{"shards":"2,7","objects":-4758}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":189,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.285279+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:24:43.285279+0000","last_peered":"2024-08-17T17:24:43.285279+0000","last_clean":"2024-08-17T17:24:43.285279+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:24:43.285279+0000","last_undegraded":"2024-08-17T17:24:43.285279+0000","last_fullsized":"2024-08-17T17:24:43.285279+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.242675+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:24:31.242675+0000","last_peered":"2024-08-17T17:24:31.242675+0000","last_clean":"2024-08-17T17:24:31.242675+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:24:31.242675+0000","last_undegraded":"2024-08-17T17:24:31.242675+0000","last_fullsized":"2024-08-17T17:24:31.242675+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"157'14350","reported_seq":14533,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.797249+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:24:43.797249+0000","last_peered":"2024-08-17T17:24:43.797249+0000","last_clean":"2024-08-17T17:24:43.797249+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:24:43.797249+0000","last_undegraded":"2024-08-17T17:24:43.797249+0000","last_fullsized":"2024-08-17T17:24:43.797249+0000","mapping_epoch":128,"log_start":"118'7356","ondisk_log_start":"118'7356","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151257088,"num_objects":2308,"num_object_clones":0,"num_object_copies":6924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14350,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.535814+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:24:13.535814+0000","last_peered":"2024-08-17T17:24:13.535814+0000","last_clean":"2024-08-17T17:24:13.535814+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:24:13.535814+0000","last_undegraded":"2024-08-17T17:24:13.535814+0000","last_fullsized":"2024-08-17T17:24:13.535814+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.242758+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:24:31.242758+0000","last_peered":"2024-08-17T17:24:31.242758+0000","last_clean":"2024-08-17T17:24:31.242758+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:24:31.242758+0000","last_undegraded":"2024-08-17T17:24:31.242758+0000","last_fullsized":"2024-08-17T17:24:31.242758+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"156'14247","reported_seq":20095,"reported_epoch":156,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:41.270713+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:24:41.270713+0000","last_peered":"2024-08-17T17:24:41.270713+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:24:41.270713+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"119'7500","ondisk_log_start":"119'7500","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159449088,"num_objects":2433,"num_object_clones":0,"num_object_copies":7299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2441,"num_objects_misplaced":2433,"num_objects_unfound":0,"num_objects_dirty":2433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14247,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2879,"num_bytes_recovered":153485312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2433},{"shards":"4,6","objects":-2441}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.243294+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:24:31.243294+0000","last_peered":"2024-08-17T17:24:31.243294+0000","last_clean":"2024-08-17T17:24:31.243294+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:24:31.243294+0000","last_undegraded":"2024-08-17T17:24:31.243294+0000","last_fullsized":"2024-08-17T17:24:31.243294+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.242880+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:24:31.242880+0000","last_peered":"2024-08-17T17:24:31.242880+0000","last_clean":"2024-08-17T17:24:31.242880+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:24:31.242880+0000","last_undegraded":"2024-08-17T17:24:31.242880+0000","last_fullsized":"2024-08-17T17:24:31.242880+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"156'14389","reported_seq":14542,"reported_epoch":156,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:41.281992+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:24:41.281992+0000","last_peered":"2024-08-17T17:24:41.281992+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:24:41.281992+0000","last_undegraded":"2024-08-17T17:24:41.281992+0000","last_fullsized":"2024-08-17T17:24:41.281992+0000","mapping_epoch":135,"log_start":"119'7641","ondisk_log_start":"119'7641","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157220864,"num_objects":2399,"num_object_clones":0,"num_object_copies":7197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2399,"num_objects_unfound":0,"num_objects_dirty":2399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14389,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":2399}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.298478+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:24:43.298478+0000","last_peered":"2024-08-17T17:24:43.298478+0000","last_clean":"2024-08-17T17:24:43.298478+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:24:43.298478+0000","last_undegraded":"2024-08-17T17:24:43.298478+0000","last_fullsized":"2024-08-17T17:24:43.298478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":154,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.278255+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:24:31.278255+0000","last_peered":"2024-08-17T17:24:31.278255+0000","last_clean":"2024-08-17T17:24:31.278255+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:24:31.278255+0000","last_undegraded":"2024-08-17T17:24:31.278255+0000","last_fullsized":"2024-08-17T17:24:31.278255+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"156'14040","reported_seq":26658,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:40.862924+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:24:40.862924+0000","last_peered":"2024-08-17T17:24:40.862924+0000","last_clean":"2024-08-17T17:24:40.862924+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:24:40.862924+0000","last_undegraded":"2024-08-17T17:24:40.862924+0000","last_fullsized":"2024-08-17T17:24:40.862924+0000","mapping_epoch":147,"log_start":"117'5920","ondisk_log_start":"117'5920","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158334976,"num_objects":2416,"num_object_clones":0,"num_object_copies":7248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14040,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":450,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.550959+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:24:13.550959+0000","last_peered":"2024-08-17T17:24:13.550959+0000","last_clean":"2024-08-17T17:24:13.550959+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:24:13.550959+0000","last_undegraded":"2024-08-17T17:24:13.550959+0000","last_fullsized":"2024-08-17T17:24:13.550959+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":683,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.274460+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:24:31.274460+0000","last_peered":"2024-08-17T17:24:31.274460+0000","last_clean":"2024-08-17T17:24:31.274460+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:24:31.274460+0000","last_undegraded":"2024-08-17T17:24:31.274460+0000","last_fullsized":"2024-08-17T17:24:31.274460+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.284910+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:24:43.284910+0000","last_peered":"2024-08-17T17:24:43.284910+0000","last_clean":"2024-08-17T17:24:43.284910+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:24:43.284910+0000","last_undegraded":"2024-08-17T17:24:43.284910+0000","last_fullsized":"2024-08-17T17:24:43.284910+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:30.717574+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:24:30.717574+0000","last_peered":"2024-08-17T17:24:30.717574+0000","last_clean":"2024-08-17T17:24:30.717574+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:24:30.717574+0000","last_undegraded":"2024-08-17T17:24:30.717574+0000","last_fullsized":"2024-08-17T17:24:30.717574+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"157'14256","reported_seq":14338,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:43.798890+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:24:43.798890+0000","last_peered":"2024-08-17T17:24:43.798890+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:24:43.798890+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"118'7256","ondisk_log_start":"118'7256","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149815296,"num_objects":2286,"num_object_clones":0,"num_object_copies":6858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14256,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-4824},{"shards":"0,1,5","objects":2286}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":528,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.245901+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:24:31.245901+0000","last_peered":"2024-08-17T17:24:31.245901+0000","last_clean":"2024-08-17T17:24:31.245901+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:24:31.245901+0000","last_undegraded":"2024-08-17T17:24:31.245901+0000","last_fullsized":"2024-08-17T17:24:31.245901+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":172,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.260771+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:24:31.260771+0000","last_peered":"2024-08-17T17:24:31.260771+0000","last_clean":"2024-08-17T17:24:31.260771+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:24:31.260771+0000","last_undegraded":"2024-08-17T17:24:31.260771+0000","last_fullsized":"2024-08-17T17:24:31.260771+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":169,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.270112+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:24:31.270112+0000","last_peered":"2024-08-17T17:24:31.270112+0000","last_clean":"2024-08-17T17:24:31.270112+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:24:31.270112+0000","last_undegraded":"2024-08-17T17:24:31.270112+0000","last_fullsized":"2024-08-17T17:24:31.270112+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"156'14055","reported_seq":14161,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:42.642088+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:24:42.642088+0000","last_peered":"2024-08-17T17:24:42.642088+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:24:42.642088+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"115'4000","ondisk_log_start":"115'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10055,"log_dups_size":0,"ondisk_log_size":10055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14055,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":2287},{"shards":"2,7","objects":-4786}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":569,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.259708+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:24:31.259708+0000","last_peered":"2024-08-17T17:24:31.259708+0000","last_clean":"2024-08-17T17:24:31.259708+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:24:31.259708+0000","last_undegraded":"2024-08-17T17:24:31.259708+0000","last_fullsized":"2024-08-17T17:24:31.259708+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.286728+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:24:43.286728+0000","last_peered":"2024-08-17T17:24:43.286728+0000","last_clean":"2024-08-17T17:24:43.286728+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:24:43.286728+0000","last_undegraded":"2024-08-17T17:24:43.286728+0000","last_fullsized":"2024-08-17T17:24:43.286728+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279047+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:24:43.279047+0000","last_peered":"2024-08-17T17:24:43.279047+0000","last_clean":"2024-08-17T17:24:43.279047+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:24:43.279047+0000","last_undegraded":"2024-08-17T17:24:43.279047+0000","last_fullsized":"2024-08-17T17:24:43.279047+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"156'13953","reported_seq":15105,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:43.062293+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:24:43.062293+0000","last_peered":"2024-08-17T17:24:43.062293+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:24:43.062293+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"118'6571","ondisk_log_start":"118'6571","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150667264,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13953,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":2299}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":970,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:43.278394+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:24:43.278394+0000","last_peered":"2024-08-17T17:24:43.278394+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:24:43.278394+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":40,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:26.248261+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:24:26.248261+0000","last_peered":"2024-08-17T17:24:26.248261+0000","last_clean":"2024-08-17T17:24:26.248261+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:24:26.248261+0000","last_undegraded":"2024-08-17T17:24:26.248261+0000","last_fullsized":"2024-08-17T17:24:26.248261+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279346+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:24:43.279346+0000","last_peered":"2024-08-17T17:24:43.279346+0000","last_clean":"2024-08-17T17:24:43.279346+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:24:43.279346+0000","last_undegraded":"2024-08-17T17:24:43.279346+0000","last_fullsized":"2024-08-17T17:24:43.279346+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"156'14195","reported_seq":27003,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.171004+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:24:43.171004+0000","last_peered":"2024-08-17T17:24:43.171004+0000","last_clean":"2024-08-17T17:24:43.171004+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:24:43.171004+0000","last_undegraded":"2024-08-17T17:24:43.171004+0000","last_fullsized":"2024-08-17T17:24:43.171004+0000","mapping_epoch":147,"log_start":"118'6833","ondisk_log_start":"118'6833","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153681920,"num_objects":2345,"num_object_clones":0,"num_object_copies":7035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14195,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.285294+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:24:43.285294+0000","last_peered":"2024-08-17T17:24:43.285294+0000","last_clean":"2024-08-17T17:24:43.285294+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:24:43.285294+0000","last_undegraded":"2024-08-17T17:24:43.285294+0000","last_fullsized":"2024-08-17T17:24:43.285294+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":455,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279528+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:24:43.279528+0000","last_peered":"2024-08-17T17:24:43.279528+0000","last_clean":"2024-08-17T17:24:43.279528+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:24:43.279528+0000","last_undegraded":"2024-08-17T17:24:43.279528+0000","last_fullsized":"2024-08-17T17:24:43.279528+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":161,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278667+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:24:43.278667+0000","last_peered":"2024-08-17T17:24:43.278667+0000","last_clean":"2024-08-17T17:24:43.278667+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:24:43.278667+0000","last_undegraded":"2024-08-17T17:24:43.278667+0000","last_fullsized":"2024-08-17T17:24:43.278667+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"156'13812","reported_seq":13961,"reported_epoch":156,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:40.672664+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:24:40.672664+0000","last_peered":"2024-08-17T17:24:40.672664+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:24:40.672664+0000","last_undegraded":"2024-08-17T17:24:40.672664+0000","last_fullsized":"2024-08-17T17:24:40.672664+0000","mapping_epoch":135,"log_start":"117'5688","ondisk_log_start":"117'5688","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161087488,"num_objects":2458,"num_object_clones":0,"num_object_copies":7374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2443,"num_objects_unfound":0,"num_objects_dirty":2458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13812,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":2458}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":158,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279484+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:24:43.279484+0000","last_peered":"2024-08-17T17:24:43.279484+0000","last_clean":"2024-08-17T17:24:43.279484+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:24:43.279484+0000","last_undegraded":"2024-08-17T17:24:43.279484+0000","last_fullsized":"2024-08-17T17:24:43.279484+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":682,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:13.551379+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:24:13.551379+0000","last_peered":"2024-08-17T17:24:13.551379+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:24:13.551379+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":154,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.261484+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:24:31.261484+0000","last_peered":"2024-08-17T17:24:31.261484+0000","last_clean":"2024-08-17T17:24:31.261484+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:24:31.261484+0000","last_undegraded":"2024-08-17T17:24:31.261484+0000","last_fullsized":"2024-08-17T17:24:31.261484+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"156'14162","reported_seq":14338,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:41.259531+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:24:41.259531+0000","last_peered":"2024-08-17T17:24:41.259531+0000","last_clean":"2024-08-17T17:24:41.259531+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:24:41.259531+0000","last_undegraded":"2024-08-17T17:24:41.259531+0000","last_fullsized":"2024-08-17T17:24:41.259531+0000","mapping_epoch":128,"log_start":"119'7475","ondisk_log_start":"119'7475","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152961024,"num_objects":2334,"num_object_clones":0,"num_object_copies":7002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14162,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":167,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.285288+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:24:43.285288+0000","last_peered":"2024-08-17T17:24:43.285288+0000","last_clean":"2024-08-17T17:24:43.285288+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:24:43.285288+0000","last_undegraded":"2024-08-17T17:24:43.285288+0000","last_fullsized":"2024-08-17T17:24:43.285288+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":650,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:43.279416+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:24:43.279416+0000","last_peered":"2024-08-17T17:24:43.279416+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:24:43.279416+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":164,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.298326+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:24:43.298326+0000","last_peered":"2024-08-17T17:24:43.298326+0000","last_clean":"2024-08-17T17:24:43.298326+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:24:43.298326+0000","last_undegraded":"2024-08-17T17:24:43.298326+0000","last_fullsized":"2024-08-17T17:24:43.298326+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"157'14274","reported_seq":15596,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.597526+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:24:43.597526+0000","last_peered":"2024-08-17T17:24:43.597526+0000","last_clean":"2024-08-17T17:24:43.597526+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:24:43.597526+0000","last_undegraded":"2024-08-17T17:24:43.597526+0000","last_fullsized":"2024-08-17T17:24:43.597526+0000","mapping_epoch":139,"log_start":"118'6730","ondisk_log_start":"118'6730","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154271744,"num_objects":2354,"num_object_clones":0,"num_object_copies":7062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14274,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":184,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.269809+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:24:31.269809+0000","last_peered":"2024-08-17T17:24:31.269809+0000","last_clean":"2024-08-17T17:24:31.269809+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:24:31.269809+0000","last_undegraded":"2024-08-17T17:24:31.269809+0000","last_fullsized":"2024-08-17T17:24:31.269809+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":399,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536320+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:24:13.536320+0000","last_peered":"2024-08-17T17:24:13.536320+0000","last_clean":"2024-08-17T17:24:13.536320+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:24:13.536320+0000","last_undegraded":"2024-08-17T17:24:13.536320+0000","last_fullsized":"2024-08-17T17:24:13.536320+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:26.248228+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:24:26.248228+0000","last_peered":"2024-08-17T17:24:26.248228+0000","last_clean":"2024-08-17T17:24:26.248228+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:24:26.248228+0000","last_undegraded":"2024-08-17T17:24:26.248228+0000","last_fullsized":"2024-08-17T17:24:26.248228+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":40,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.242465+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:24:31.242465+0000","last_peered":"2024-08-17T17:24:31.242465+0000","last_clean":"2024-08-17T17:24:31.242465+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:24:31.242465+0000","last_undegraded":"2024-08-17T17:24:31.242465+0000","last_fullsized":"2024-08-17T17:24:31.242465+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":50,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.242576+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:24:31.242576+0000","last_peered":"2024-08-17T17:24:31.242576+0000","last_clean":"2024-08-17T17:24:31.242576+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:24:31.242576+0000","last_undegraded":"2024-08-17T17:24:31.242576+0000","last_fullsized":"2024-08-17T17:24:31.242576+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":150,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278516+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:24:43.278516+0000","last_peered":"2024-08-17T17:24:43.278516+0000","last_clean":"2024-08-17T17:24:43.278516+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:24:43.278516+0000","last_undegraded":"2024-08-17T17:24:43.278516+0000","last_fullsized":"2024-08-17T17:24:43.278516+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":150,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551184+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:24:13.551184+0000","last_peered":"2024-08-17T17:24:13.551184+0000","last_clean":"2024-08-17T17:24:13.551184+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:24:13.551184+0000","last_undegraded":"2024-08-17T17:24:13.551184+0000","last_fullsized":"2024-08-17T17:24:13.551184+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":84,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.551294+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:24:13.551294+0000","last_peered":"2024-08-17T17:24:13.551294+0000","last_clean":"2024-08-17T17:24:13.551294+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:24:13.551294+0000","last_undegraded":"2024-08-17T17:24:13.551294+0000","last_fullsized":"2024-08-17T17:24:13.551294+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.283614+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:24:43.283614+0000","last_peered":"2024-08-17T17:24:43.283614+0000","last_clean":"2024-08-17T17:24:43.283614+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:24:43.283614+0000","last_undegraded":"2024-08-17T17:24:43.283614+0000","last_fullsized":"2024-08-17T17:24:43.283614+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.267749+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:24:31.267749+0000","last_peered":"2024-08-17T17:24:31.267749+0000","last_clean":"2024-08-17T17:24:31.267749+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:24:31.267749+0000","last_undegraded":"2024-08-17T17:24:31.267749+0000","last_fullsized":"2024-08-17T17:24:31.267749+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":168,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.298250+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:24:43.298250+0000","last_peered":"2024-08-17T17:24:43.298250+0000","last_clean":"2024-08-17T17:24:43.298250+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:24:43.298250+0000","last_undegraded":"2024-08-17T17:24:43.298250+0000","last_fullsized":"2024-08-17T17:24:43.298250+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.716627+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:24:20.716627+0000","last_peered":"2024-08-17T17:24:20.716627+0000","last_clean":"2024-08-17T17:24:20.716627+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:24:20.716627+0000","last_undegraded":"2024-08-17T17:24:20.716627+0000","last_fullsized":"2024-08-17T17:24:20.716627+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.298399+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:24:43.298399+0000","last_peered":"2024-08-17T17:24:43.298399+0000","last_clean":"2024-08-17T17:24:43.298399+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:24:43.298399+0000","last_undegraded":"2024-08-17T17:24:43.298399+0000","last_fullsized":"2024-08-17T17:24:43.298399+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":158,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.270036+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:24:31.270036+0000","last_peered":"2024-08-17T17:24:31.270036+0000","last_clean":"2024-08-17T17:24:31.270036+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:24:31.270036+0000","last_undegraded":"2024-08-17T17:24:31.270036+0000","last_fullsized":"2024-08-17T17:24:31.270036+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278043+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:24:43.278043+0000","last_peered":"2024-08-17T17:24:43.278043+0000","last_clean":"2024-08-17T17:24:43.278043+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:24:43.278043+0000","last_undegraded":"2024-08-17T17:24:43.278043+0000","last_fullsized":"2024-08-17T17:24:43.278043+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":181,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.285027+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:24:43.285027+0000","last_peered":"2024-08-17T17:24:43.285027+0000","last_clean":"2024-08-17T17:24:43.285027+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:24:43.285027+0000","last_undegraded":"2024-08-17T17:24:43.285027+0000","last_fullsized":"2024-08-17T17:24:43.285027+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":39,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536154+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:24:13.536154+0000","last_peered":"2024-08-17T17:24:13.536154+0000","last_clean":"2024-08-17T17:24:13.536154+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:24:13.536154+0000","last_undegraded":"2024-08-17T17:24:13.536154+0000","last_fullsized":"2024-08-17T17:24:13.536154+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":187,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.281899+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:24:31.281899+0000","last_peered":"2024-08-17T17:24:31.281899+0000","last_clean":"2024-08-17T17:24:31.281899+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:24:31.281899+0000","last_undegraded":"2024-08-17T17:24:31.281899+0000","last_fullsized":"2024-08-17T17:24:31.281899+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":164,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.261401+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:24:31.261401+0000","last_peered":"2024-08-17T17:24:31.261401+0000","last_clean":"2024-08-17T17:24:31.261401+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:24:31.261401+0000","last_undegraded":"2024-08-17T17:24:31.261401+0000","last_fullsized":"2024-08-17T17:24:31.261401+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":39,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.535877+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:24:13.535877+0000","last_peered":"2024-08-17T17:24:13.535877+0000","last_clean":"2024-08-17T17:24:13.535877+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:24:13.535877+0000","last_undegraded":"2024-08-17T17:24:13.535877+0000","last_fullsized":"2024-08-17T17:24:13.535877+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":140,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536495+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:24:13.536495+0000","last_peered":"2024-08-17T17:24:13.536495+0000","last_clean":"2024-08-17T17:24:13.536495+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:24:13.536495+0000","last_undegraded":"2024-08-17T17:24:13.536495+0000","last_fullsized":"2024-08-17T17:24:13.536495+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":39,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536041+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:24:13.536041+0000","last_peered":"2024-08-17T17:24:13.536041+0000","last_clean":"2024-08-17T17:24:13.536041+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:24:13.536041+0000","last_undegraded":"2024-08-17T17:24:13.536041+0000","last_fullsized":"2024-08-17T17:24:13.536041+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":125,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.536250+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:24:13.536250+0000","last_peered":"2024-08-17T17:24:13.536250+0000","last_clean":"2024-08-17T17:24:13.536250+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:24:13.536250+0000","last_undegraded":"2024-08-17T17:24:13.536250+0000","last_fullsized":"2024-08-17T17:24:13.536250+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":186,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.284744+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:24:43.284744+0000","last_peered":"2024-08-17T17:24:43.284744+0000","last_clean":"2024-08-17T17:24:43.284744+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:24:43.284744+0000","last_undegraded":"2024-08-17T17:24:43.284744+0000","last_fullsized":"2024-08-17T17:24:43.284744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":174,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.285658+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:24:43.285658+0000","last_peered":"2024-08-17T17:24:43.285658+0000","last_clean":"2024-08-17T17:24:43.285658+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:24:43.285658+0000","last_undegraded":"2024-08-17T17:24:43.285658+0000","last_fullsized":"2024-08-17T17:24:43.285658+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":39,"reported_epoch":155,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.535403+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:24:13.535403+0000","last_peered":"2024-08-17T17:24:13.535403+0000","last_clean":"2024-08-17T17:24:13.535403+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:24:13.535403+0000","last_undegraded":"2024-08-17T17:24:13.535403+0000","last_fullsized":"2024-08-17T17:24:13.535403+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":175,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:24:31.267590+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:24:31.267590+0000","last_peered":"2024-08-17T17:24:31.267590+0000","last_clean":"2024-08-17T17:24:31.267590+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:24:31.267590+0000","last_undegraded":"2024-08-17T17:24:31.267590+0000","last_fullsized":"2024-08-17T17:24:31.267590+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2458648600,"num_objects":37517,"num_object_clones":0,"num_object_copies":112551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35554,"num_objects_misplaced":7275,"num_objects_unfound":0,"num_objects_dirty":37517,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":226367,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17888,"num_bytes_recovered":1006698496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8962191360,"data_stored":8962179144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131097,"ondisk_log_size":131097,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201534,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902044,"kb_used_data":776596,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":92843108,"statfs":{"total":95995035648,"available":95071342592,"internally_reserved":0,"allocated":795234304,"data_stored":794335993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":128448971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463172,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":746384,"kb_used_data":634888,"kb_used_omap":1,"kb_used_meta":111486,"kb_avail":92998768,"statfs":{"total":95995035648,"available":95230738432,"internally_reserved":0,"allocated":650125312,"data_stored":649239455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":114162119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724808,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986444,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456912,"kb_used_data":1255176,"kb_used_omap":1,"kb_used_meta":201726,"kb_avail":92288240,"statfs":{"total":95995035648,"available":94503157760,"internally_reserved":0,"allocated":1285300224,"data_stored":1284413571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":206567878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435336,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691668,"kb_used_data":591944,"kb_used_omap":5,"kb_used_meta":99706,"kb_avail":93053484,"statfs":{"total":95995035648,"available":95286767616,"internally_reserved":0,"allocated":606150656,"data_stored":605267799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":102099017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598862,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051588,"kb_used_data":893564,"kb_used_omap":7,"kb_used_meta":158008,"kb_avail":92693564,"statfs":{"total":95995035648,"available":94918209536,"internally_reserved":0,"allocated":915009536,"data_stored":914130921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":161800365},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762389,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":867832,"kb_used_data":745904,"kb_used_omap":7,"kb_used_meta":121912,"kb_avail":92877320,"statfs":{"total":95995035648,"available":95106375680,"internally_reserved":0,"allocated":763805696,"data_stored":762938475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":124838305},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925915,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1546632,"kb_used_data":1344128,"kb_used_omap":7,"kb_used_meta":202424,"kb_avail":92198520,"statfs":{"total":95995035648,"available":94411284480,"internally_reserved":0,"allocated":1376387072,"data_stored":1375505142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":207282780},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1360592896,"data_stored":1360592896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":749469696,"data_stored":749469696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":908136448,"data_stored":908132376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":604176384,"data_stored":604176384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1273036800,"data_stored":1273036800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2645692416,"data_stored":2645688344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":639631360,"data_stored":639631360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":781455360,"data_stored":781451288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:48.208 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:48.215 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:48.240 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:48.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:48.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[104516]: from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:48.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[104516]: pgmap v471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 35684/109008 objects degraded (32.735%); 6684/109008 objects misplaced (6.132%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:48.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35684/109008 objects degraded (32.735%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:48.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[109246]: from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:48.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[109246]: pgmap v471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 35684/109008 objects degraded (32.735%); 6684/109008 objects misplaced (6.132%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:48.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35684/109008 objects degraded (32.735%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:48.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:48.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:48.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:48.531 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:48.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:48.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:48 smithi175 ceph-mon[101434]: from='client.34884 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:48 smithi175 ceph-mon[101434]: pgmap v471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 35684/109008 objects degraded (32.735%); 6684/109008 objects misplaced (6.132%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35684/109008 objects degraded (32.735%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:48.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:48.777 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:48.878 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:48.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:49.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:49.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:49.181 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:49.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:49.309 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:49.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:49.440 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:49.542 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:49.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:49.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:49.669 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:49.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:49.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:49.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:49.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:49.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:49.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:50.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:50.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:50.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:50.245 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:50.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:50.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:50.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:50.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:50.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:50.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:50.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:50.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:50 smithi175 ceph-mon[101434]: pgmap v472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35684/103146 objects degraded (34.596%); 6684/103146 objects misplaced (6.480%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:50.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:50 smithi064 ceph-mon[104516]: pgmap v472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35684/103146 objects degraded (34.596%); 6684/103146 objects misplaced (6.480%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:50.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:50 smithi064 ceph-mon[109246]: pgmap v472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35684/103146 objects degraded (34.596%); 6684/103146 objects misplaced (6.480%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:50.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:50.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:50.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.930 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:50.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:51.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:51.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:51.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:51.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:51.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:51.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:51.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:51.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:51.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:51.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:51.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:51.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:51.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:51.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:51.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:51.870 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:51.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:52.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:52.032 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:52.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:52.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:52.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:52.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:52.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:52.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:52.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:52 smithi064 ceph-mon[104516]: pgmap v473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35684/102534 objects degraded (34.802%); 6481/102534 objects misplaced (6.321%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:52.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:52 smithi064 ceph-mon[109246]: pgmap v473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35684/102534 objects degraded (34.802%); 6481/102534 objects misplaced (6.321%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:52.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:52.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:52 smithi175 ceph-mon[101434]: pgmap v473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35684/102534 objects degraded (34.802%); 6481/102534 objects misplaced (6.321%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:52.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:52.713 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:24:52.713 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:24:52.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:24:52.713 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:24:52.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131921 5.a 5:5beda773:::benchmark_data_smithi064_147470_object131920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 170.666251862, 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4918728809999999, 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.714 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131921 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042045+0000", 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9999999999999997e-07 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042067+0000", 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1823000000000001e-05 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042076+0000", 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9230000000000008e-06 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042082+0000", 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8850000000000004e-06 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.045641+0000", 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035593339999999999 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.049865+0000", 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042238900000000001 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050039+0000", 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017439300000000001 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.065709+0000", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015669374999999999 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.065715+0000", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5409999999999997e-06 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040390+0000", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97467535299999997 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040408+0000", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7765e-05 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.507316+0000", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.466907371 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.509272+0000", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019564259999999998 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533838+0000", 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024566119000000001 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533917+0000", 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9083000000000002e-05 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131926 5.a 5:50f30e39:::benchmark_data_smithi064_147470_object131925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:24:52.721 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 170.647114915, 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4772495339999998, 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131926 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061183+0000", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6920000000000001e-06 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061208+0000", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4828999999999999e-05 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061218+0000", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560000000000001e-05 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061225+0000", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0659999999999998e-06 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.066388+0000", 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005162725 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.066397+0000", 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1779999999999992e-06 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040484+0000", 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97408712900000005 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040495+0000", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0538e-05 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515768+0000", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.47527309 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515914+0000", 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014613699999999999 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.516046+0000", 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013153899999999999 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533966+0000", 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017920177999999998 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534025+0000", 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8739000000000001e-05 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534037+0000", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1953000000000001e-05 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534069+0000", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1854999999999999e-05 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.538431+0000", 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043623259999999997 2024-08-17T17:24:52.729 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140754 5.a 5:5532459e:::benchmark_data_smithi064_147470_object8810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.129189+0000", 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.57910695199999, 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.150800482, 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140754 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129189+0000", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129189+0000", 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129190+0000", 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2e-07 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129192+0000", 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2479999999999999e-06 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129198+0000", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6119999999999998e-06 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.129203+0000", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3020000000000002e-06 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140681+0000", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011477675999999999 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140840+0000", 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015908099999999999 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140996+0000", 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015614900000000001 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155177+0000", 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014181601 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.171990+0000", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016812678000000001 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230591+0000", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058600897999999998 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230602+0000", 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1429e-05 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.267908+0000", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0373052140000001 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.268049+0000", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014184500000000001 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.279900+0000", 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011850272 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.279990+0000", 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0056999999999995e-05 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140758 5.a 5:5403018b:::benchmark_data_smithi064_147470_object8814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.56794344100001, 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234468573, 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.737 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140758 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140354+0000", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6130000000000001e-06 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140360+0000", 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6579999999999999e-06 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140377+0000", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7433000000000001e-05 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140386+0000", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8079999999999994e-06 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145573+0000", 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051867600000000003 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145756+0000", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018317 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145928+0000", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017162699999999999 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155183+0000", 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092555829999999995 2024-08-17T17:24:52.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172033+0000", 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016849670000000001 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230629+0000", 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058595582 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230636+0000", 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6550000000000004e-06 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.268659+0000", 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.038022794 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269045+0000", 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000385907 2024-08-17T17:24:52.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374769+0000", 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10572400799999999 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374821+0000", 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2305000000000002e-05 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140763 5.a 5:51aa539c:::benchmark_data_smithi064_147470_object8819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.56789542000001, 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234340475, 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140763 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8100000000000003e-07 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140404+0000", 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3369999999999998e-06 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140410+0000", 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0979999999999999e-06 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140414+0000", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7809999999999996e-06 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150277+0000", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098622109999999992 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150431+0000", 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154711 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150571+0000", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013928700000000001 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172074+0000", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02150297 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172168+0000", 2024-08-17T17:24:52.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4524999999999994e-05 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230661+0000", 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058492423000000002 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230668+0000", 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.063e-06 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269162+0000", 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0384945990000001 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269387+0000", 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022499099999999999 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374683+0000", 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.105295499 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374741+0000", 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8498999999999997e-05 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140766 5.a 5:5069331d:::benchmark_data_smithi064_147470_object8822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.553246917, 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.219602018, 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.752 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140766 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155050+0000", 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9399999999999995e-07 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155052+0000", 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5009999999999999e-06 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155059+0000", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4609999999999996e-06 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155064+0000", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2769999999999996e-06 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.162992+0000", 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079281260000000006 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.163140+0000", 2024-08-17T17:24:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:52.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014762 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.163297+0000", 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015770199999999999 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193553+0000", 2024-08-17T17:24:52.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030255414000000001 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.198428+0000", 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048752889999999997 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230694+0000", 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032266112999999999 2024-08-17T17:24:52.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230701+0000", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1359999999999999e-06 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269493+0000", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.038791429 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269585+0000", 2024-08-17T17:24:52.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2304000000000003e-05 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374585+0000", 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.104999962 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374651+0000", 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6190000000000002e-05 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.761 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140776 5.a 5:596336eb:::benchmark_data_smithi064_147470_object8832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.53594329399999, 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.201320121, 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.762 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140776 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172355+0000", 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9470000000000002e-06 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172360+0000", 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4099999999999999e-06 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172375+0000", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4679e-05 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172382+0000", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172445+0000", 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2534999999999997e-05 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172619+0000", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173707 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172772+0000", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015342599999999999 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193595+0000", 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020822811 2024-08-17T17:24:52.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.198437+0000", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00484199 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230726+0000", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032288692000000001 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230733+0000", 2024-08-17T17:24:52.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2599999999999999e-06 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269619+0000", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.03888622 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269709+0000", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0048000000000002e-05 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.373587+0000", 2024-08-17T17:24:52.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103877517 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.373673+0000", 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6349999999999998e-05 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140781 5.a 5:5cbf5a79:::benchmark_data_smithi064_147470_object8837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:24:52.769 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.525110619, 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1908856889999999, 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140781 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183187+0000", 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.065e-06 2024-08-17T17:24:52.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183191+0000", 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6700000000000002e-06 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183200+0000", 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2050000000000002e-06 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183205+0000", 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1080000000000001e-06 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193625+0000", 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010420674 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193810+0000", 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000184721 2024-08-17T17:24:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193983+0000", 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173072 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.199492+0000", 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055087130000000002 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.200443+0000", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000951205 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230758+0000", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030315023999999999 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230765+0000", 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3130000000000001e-06 2024-08-17T17:24:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269855+0000", 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.039089626 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270151+0000", 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000296497 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374011+0000", 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103859375 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374071+0000", 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0421000000000003e-05 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140789 5.a 5:56fb2894:::benchmark_data_smithi064_147470_object8845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:24:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.201268+0000", 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.50702815299999, 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173262982, 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140789 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201268+0000", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201268+0000", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201269+0000", 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9800000000000004e-07 2024-08-17T17:24:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201271+0000", 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7539999999999998e-06 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201277+0000", 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9699999999999996e-06 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201282+0000", 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5210000000000004e-06 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.201978+0000", 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069644399999999995 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.202138+0000", 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015989800000000001 2024-08-17T17:24:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.202328+0000", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018994699999999999 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230790+0000", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0284616 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230797+0000", 2024-08-17T17:24:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270448+0000", 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0396510779999999 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270562+0000", 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011429000000000001 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374420+0000", 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103857699 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374531+0000", 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011137399999999999 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140793 5.a 5:56d308a9:::benchmark_data_smithi064_147470_object8849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.238351+0000", 2024-08-17T17:24:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.46994481499999, 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1340820709999999, 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140793 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238351+0000", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238351+0000", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238353+0000", 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1030000000000001e-06 2024-08-17T17:24:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238356+0000", 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9480000000000001e-06 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238368+0000", 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1321000000000001e-05 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238373+0000", 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2320000000000001e-06 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238436+0000", 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3133999999999996e-05 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.238448+0000", 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2153999999999999e-05 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270603+0000", 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0321548149999999 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270737+0000", 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013378899999999999 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270904+0000", 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016759300000000001 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.279427+0000", 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085226130000000001 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280032+0000", 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060520300000000001 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372150+0000", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.092118033000000002 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372213+0000", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2676999999999996e-05 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372433+0000", 2024-08-17T17:24:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000220456 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140797 5.a 5:581a1a91:::benchmark_data_smithi064_147470_object8853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.254236+0000", 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 152.45406065500001, 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.118403627, 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140797 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254236+0000", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254236+0000", 2024-08-17T17:24:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254237+0000", 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.886e-06 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254244+0000", 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.861e-06 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254257+0000", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2503e-05 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254265+0000", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8760000000000005e-06 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254366+0000", 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010118 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254378+0000", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2325e-05 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275168+0000", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.020790031 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275299+0000", 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013088999999999999 2024-08-17T17:24:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.275426+0000", 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000126857 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.280238+0000", 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048123200000000001 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282598+0000", 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002359753 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372475+0000", 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.089877259000000001 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372523+0000", 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7488999999999999e-05 2024-08-17T17:24:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.372639+0000", 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116397 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145193 5.a 5:5d5d0224:::benchmark_data_smithi064_147470_object13249:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.048514+0000", 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.65978173400001, 2024-08-17T17:24:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0129290799999999, 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145193 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048514+0000", 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048514+0000", 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048516+0000", 2024-08-17T17:24:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.739e-06 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048523+0000", 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.939e-06 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048628+0000", 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010450600000000001 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.048639+0000", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1124999999999999e-05 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.049396+0000", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00075763000000000002 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.049537+0000", 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014075000000000001 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.049682+0000", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014435000000000001 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.058835+0000", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091530740000000006 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.058843+0000", 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5499999999999995e-06 2024-08-17T17:24:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.061361+0000", 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0025180600000001 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.061444+0000", 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2356999999999998e-05 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145257 5.a 5:5d080b25:::benchmark_data_smithi064_147470_object13313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.134778+0000", 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.57351784900001, 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93670415399999996, 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145257 2024-08-17T17:24:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134778+0000", 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134778+0000", 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134780+0000", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.48e-06 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134786+0000", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8379999999999998e-06 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134798+0000", 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2639999999999999e-05 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134806+0000", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7710000000000006e-06 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134918+0000", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112293 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.134930+0000", 2024-08-17T17:24:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1257000000000001e-05 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.061845+0000", 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92691502800000003 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.061954+0000", 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109063 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.062027+0000", 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2960000000000006e-05 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071292+0000", 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092649039999999992 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071384+0000", 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2712000000000003e-05 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071413+0000", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8853999999999999e-05 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071445+0000", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1393999999999998e-05 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071483+0000", 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7960000000000002e-05 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145285 5.a 5:5f38eac0:::benchmark_data_smithi064_147470_object13341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.164821+0000", 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.54347482200001, 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90684881900000003, 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145285 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164821+0000", 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164821+0000", 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164823+0000", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4929999999999999e-06 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164828+0000", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8890000000000001e-06 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164839+0000", 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1358999999999999e-05 2024-08-17T17:24:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164846+0000", 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0600000000000002e-06 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164929+0000", 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2410999999999994e-05 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.164938+0000", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6719999999999999e-06 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.065143+0000", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90020492500000004 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.065261+0000", 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011799900000000001 2024-08-17T17:24:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.065385+0000", 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012361100000000001 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071543+0000", 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061578700000000002 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071560+0000", 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7555999999999999e-05 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071615+0000", 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5148000000000001e-05 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071647+0000", 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1470999999999997e-05 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071670+0000", 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3354999999999999e-05 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145292 5.a 5:52b74139:::benchmark_data_smithi064_147470_object13348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.170854+0000", 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.53744256100001, 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90098531000000004, 2024-08-17T17:24:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145292 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170854+0000", 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170854+0000", 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170854+0000", 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.89e-07 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170856+0000", 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3300000000000001e-06 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170862+0000", 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8619999999999999e-06 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.170867+0000", 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.887e-06 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.172218+0000", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001351056 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.172228+0000", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7079999999999992e-06 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.068418+0000", 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89618964999999995 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.068513+0000", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4875000000000002e-05 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.068579+0000", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6452999999999995e-05 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071567+0000", 2024-08-17T17:24:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029885290000000002 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071726+0000", 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015802700000000001 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071772+0000", 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6820000000000002e-05 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071807+0000", 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4258999999999998e-05 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071839+0000", 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2364999999999999e-05 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145295 5.a 5:5857b96c:::benchmark_data_smithi064_147470_object13351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.178461+0000", 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.52983539100001, 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90906571400000002, 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145295 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178461+0000", 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178461+0000", 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178462+0000", 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.449e-06 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178467+0000", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8080000000000004e-06 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178482+0000", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5274e-05 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178489+0000", 2024-08-17T17:24:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.007e-06 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178548+0000", 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8397999999999999e-05 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.178557+0000", 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4110000000000002e-06 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071872+0000", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89331501899999999 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.071995+0000", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122829 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.072097+0000", 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102416 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.079831+0000", 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077335349999999997 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.079847+0000", 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.588e-05 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087429+0000", 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075822440000000001 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087479+0000", 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0135e-05 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087527+0000", 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7308999999999997e-05 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145301 5.a 5:5af9acb1:::benchmark_data_smithi064_147470_object13357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.180536+0000", 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.52776032, 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90717283100000001, 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145301 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180536+0000", 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180536+0000", 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180536+0000", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3500000000000002e-07 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180538+0000", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0940000000000002e-06 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180545+0000", 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1330000000000004e-06 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180549+0000", 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8269999999999997e-06 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180620+0000", 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0115000000000002e-05 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.180628+0000", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9420000000000001e-06 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.076022+0000", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89539401100000005 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.076153+0000", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130266 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.076285+0000", 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132681 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087376+0000", 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011091021 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087537+0000", 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000160707 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087628+0000", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0581999999999994e-05 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087675+0000", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7639000000000002e-05 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087709+0000", 2024-08-17T17:24:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3377999999999998e-05 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145311 5.a 5:5cb79744:::benchmark_data_smithi064_147470_object13367:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.194725+0000", 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.513571598, 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89317816500000002, 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145311 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194725+0000", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194725+0000", 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194725+0000", 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2900000000000002e-07 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194730+0000", 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1849999999999997e-06 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194740+0000", 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0345e-05 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194748+0000", 2024-08-17T17:24:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7430000000000002e-06 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194838+0000", 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0017000000000002e-05 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.194847+0000", 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.662e-06 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.079870+0000", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88502255600000002 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.079991+0000", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121148 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.080106+0000", 2024-08-17T17:24:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011505099999999999 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087543+0000", 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007436443 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087583+0000", 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0283e-05 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087817+0000", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000234362 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087861+0000", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3640999999999998e-05 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087903+0000", 2024-08-17T17:24:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1900000000000002e-05 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145312 5.a 5:59036b96:::benchmark_data_smithi064_147470_object13368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.195222+0000", 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.513074467, 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89283980399999996, 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145312 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195222+0000", 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195222+0000", 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195222+0000", 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4300000000000002e-07 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195226+0000", 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9129999999999996e-06 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195241+0000", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4406000000000001e-05 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195248+0000", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1690000000000003e-06 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195301+0000", 2024-08-17T17:24:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3211000000000003e-05 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.195310+0000", 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6060000000000003e-06 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.083316+0000", 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88800585899999995 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.083418+0000", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102532 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.083536+0000", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117769 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087719+0000", 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041825919999999997 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087767+0000", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8674000000000002e-05 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.087948+0000", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000180409 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.088029+0000", 2024-08-17T17:24:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1451000000000003e-05 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.088062+0000", 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2469999999999999e-05 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:145319 5.a 5:5a25688a:::benchmark_data_smithi064_147470_object13375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:27.200871+0000", 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.50742484099999, 2024-08-17T17:24:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90659111100000001, 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145319 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200871+0000", 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:24:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200871+0000", 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200872+0000", 2024-08-17T17:24:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0899999999999999e-06 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200878+0000", 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1630000000000004e-06 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200901+0000", 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3235e-05 2024-08-17T17:24:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200909+0000", 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2460000000000003e-06 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200966+0000", 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7367000000000001e-05 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:24:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:27.200977+0000", 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0216e-05 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.088093+0000", 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88711635899999997 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.088180+0000", 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7416999999999993e-05 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.088272+0000", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1189999999999999e-05 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.098289+0000", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010017043 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.098300+0000", 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1060000000000001e-05 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.107318+0000", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090183599999999996 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.107415+0000", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7157000000000005e-05 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:28.107462+0000", 2024-08-17T17:24:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7207999999999999e-05 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:24:52.867 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:24:52.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:52.924 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:53.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:53.049 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:53.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:53.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:53.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:53.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:53.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:53.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:53.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:53.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35684/102534 objects degraded (34.802%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:53.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35684/102534 objects degraded (34.802%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:53.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35684/102534 objects degraded (34.802%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:53.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:53.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:53.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:53.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:53.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:24:53.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:54.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:54.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:54.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:54.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:54.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:54.317 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:24:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:24:53] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:24:54.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:54.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:54.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:54.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:54.619 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:54 smithi175 ceph-mon[101434]: pgmap v474: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 35780/96282 objects degraded (37.162%); 6045/96282 objects misplaced (6.278%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:54 smithi064 ceph-mon[104516]: pgmap v474: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 35780/96282 objects degraded (37.162%); 6045/96282 objects misplaced (6.278%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:54 smithi064 ceph-mon[109246]: pgmap v474: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.0 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 35780/96282 objects degraded (37.162%); 6045/96282 objects misplaced (6.278%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:24:54.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:54.762 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:54.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:54.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:54.887 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:54.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:55.045 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:55.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:55.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:55.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:55.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:55.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:55.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:55.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:55.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:55.823 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.857 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:55.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:55.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:55.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:55.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:56.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:56.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[104516]: pgmap v475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 35780/93351 objects degraded (38.328%); 6045/93351 objects misplaced (6.476%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[109246]: pgmap v475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 35780/93351 objects degraded (38.328%); 6045/93351 objects misplaced (6.476%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:56.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:56.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:56.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:56.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:56 smithi175 ceph-mon[101434]: pgmap v475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.9 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 35780/93351 objects degraded (38.328%); 6045/93351 objects misplaced (6.476%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:24:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:24:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:56.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:56.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:56.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:56.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:56.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:56.816 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:56.914 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:56.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:56.979 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:57.048 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:24:57.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:57.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:57.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:57.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1371722602' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:57 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1371722602' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:57 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:57.259 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:57.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:57.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1371722602' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:57 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:57.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:57.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:57.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:57.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:57.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:57.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:57.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:58.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:58.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:58.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[104516]: pgmap v476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35851/89793 objects degraded (39.926%); 5459/89793 objects misplaced (6.080%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[104516]: osdmap e158: 8 total, 8 up, 7 in 2024-08-17T17:24:58.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[109246]: pgmap v476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35851/89793 objects degraded (39.926%); 5459/89793 objects misplaced (6.080%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:58 smithi064 ceph-mon[109246]: osdmap e158: 8 total, 8 up, 7 in 2024-08-17T17:24:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:58.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:58 smithi175 ceph-mon[101434]: pgmap v476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.8 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 35851/89793 objects degraded (39.926%); 5459/89793 objects misplaced (6.080%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:58 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:58 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:58 smithi175 ceph-mon[101434]: osdmap e158: 8 total, 8 up, 7 in 2024-08-17T17:24:58.491 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:58.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:58.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:58.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:58.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:58.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:58.919 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:24:58.919 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:58.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.961 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:24:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:58.998 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:24:58.999 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:59.038 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:24:59.038 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:59.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:59.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:24:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:59.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:59.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:59.117 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:24:59.118 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:59.152 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:24:59.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:59.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:59 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35851/89793 objects degraded (39.926%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:59.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:24:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1151733083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:59.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:59 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35851/89793 objects degraded (39.926%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:59.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:24:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1151733083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:59.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:24:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:59.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:59.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:59.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:59.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:59 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35851/89793 objects degraded (39.926%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:24:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:24:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1151733083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:59.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:59.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:59.632 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:59.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:24:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:59.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:59.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:59.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:59.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:59.937 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:24:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:00.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:00.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:00.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:00.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:00.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:00.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:00.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:00.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:00.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:25:00.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:00 smithi175 ceph-mon[101434]: pgmap v478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 35851/85146 objects degraded (42.105%); 5459/85146 objects misplaced (6.411%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:00.434 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:00 smithi064 ceph-mon[104516]: pgmap v478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 35851/85146 objects degraded (42.105%); 5459/85146 objects misplaced (6.411%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:00.434 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:00 smithi064 ceph-mon[109246]: pgmap v478: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 35851/85146 objects degraded (42.105%); 5459/85146 objects misplaced (6.411%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:00.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:00.482 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:00.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:00.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:00.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:00.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:00.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:00.896 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:00.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:01.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:01.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:01.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:01.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:01.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:01.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:01.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:01.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:01.587 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:01.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:01.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:01.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:01.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:01.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:02.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:02.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:02.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:02.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:02.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:02.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:02.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:02 smithi175 ceph-mon[101434]: pgmap v479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 35851/83343 objects degraded (43.016%); 5234/83343 objects misplaced (6.280%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:02.419 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:02 smithi064 ceph-mon[104516]: pgmap v479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 35851/83343 objects degraded (43.016%); 5234/83343 objects misplaced (6.280%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:02.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:02 smithi064 ceph-mon[109246]: pgmap v479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.7 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 35851/83343 objects degraded (43.016%); 5234/83343 objects misplaced (6.280%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:25:02.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:02.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:02.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:02.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:02.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:02.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:02.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:02.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:02.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:03.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.136 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:03.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:03.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:03.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:03.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:03.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:03.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:03.507 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:03.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:03.563 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:03.564 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:03.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:03.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:03.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:03.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:03.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:03.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:03.852 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:03.853 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:25:03.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:03.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:04.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:04.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:04.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:04.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:04.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:04.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:04.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:04.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:04 smithi064 ceph-mon[104516]: pgmap v480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.6 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 35918/80241 objects degraded (44.763%); 4836/80241 objects misplaced (6.027%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:04.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:04 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35918/80241 objects degraded (44.763%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:04.401 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:03] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:25:04.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:04 smithi064 ceph-mon[109246]: pgmap v480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.6 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 35918/80241 objects degraded (44.763%); 4836/80241 objects misplaced (6.027%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:04.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:04 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35918/80241 objects degraded (44.763%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:04.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:04 smithi175 ceph-mon[101434]: pgmap v480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.6 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 35918/80241 objects degraded (44.763%); 4836/80241 objects misplaced (6.027%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:04 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35918/80241 objects degraded (44.763%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:04.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:25:04.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:04.525 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:04.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:04.681 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:04.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:04.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:04.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:04.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:04.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:05.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:05.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:05.128 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":480,"stamp":"2024-08-17T17:25:02.856045+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1755781117,"num_objects":26747,"num_object_clones":0,"num_object_copies":80241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35918,"num_objects_misplaced":4836,"num_objects_unfound":0,"num_objects_dirty":26747,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":238288,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18431,"num_bytes_recovered":1043602596,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132277,"ondisk_log_size":132277,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5520104,"kb_used_data":4467112,"kb_used_omap":33,"kb_used_meta":1052702,"kb_avail":650695960,"statfs":{"total":671965249536,"available":666312663040,"internally_reserved":0,"allocated":4574322688,"data_stored":4568039090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1077967435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,5,7,12,7,3,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-350420992,"num_objects":-5347,"num_object_clones":0,"num_object_copies":-16041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":138,"num_objects_misplaced":-1209,"num_objects_unfound":0,"num_objects_dirty":-5347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5347,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":244,"num_bytes_recovered":15990784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003710"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.074422+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:24:57.074422+0000","last_peered":"2024-08-17T17:24:57.074422+0000","last_clean":"2024-08-17T17:24:57.074422+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:24:57.074422+0000","last_undegraded":"2024-08-17T17:24:57.074422+0000","last_fullsized":"2024-08-17T17:24:57.074422+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":152,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.280792+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:24:43.280792+0000","last_peered":"2024-08-17T17:24:43.280792+0000","last_clean":"2024-08-17T17:24:43.280792+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:24:43.280792+0000","last_undegraded":"2024-08-17T17:24:43.280792+0000","last_fullsized":"2024-08-17T17:24:43.280792+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":95,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.095006+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:24:57.095006+0000","last_peered":"2024-08-17T17:24:57.095006+0000","last_clean":"2024-08-17T17:24:57.095006+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:24:57.095006+0000","last_undegraded":"2024-08-17T17:24:57.095006+0000","last_fullsized":"2024-08-17T17:24:57.095006+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080367+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:24:57.080367+0000","last_peered":"2024-08-17T17:24:57.080367+0000","last_clean":"2024-08-17T17:24:57.080367+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:24:57.080367+0000","last_undegraded":"2024-08-17T17:24:57.080367+0000","last_fullsized":"2024-08-17T17:24:57.080367+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":162,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080129+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:24:57.080129+0000","last_peered":"2024-08-17T17:24:57.080129+0000","last_clean":"2024-08-17T17:24:57.080129+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:24:57.080129+0000","last_undegraded":"2024-08-17T17:24:57.080129+0000","last_fullsized":"2024-08-17T17:24:57.080129+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.280665+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:24:43.280665+0000","last_peered":"2024-08-17T17:24:43.280665+0000","last_clean":"2024-08-17T17:24:43.280665+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:24:43.280665+0000","last_undegraded":"2024-08-17T17:24:43.280665+0000","last_fullsized":"2024-08-17T17:24:43.280665+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.075381+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:24:57.075381+0000","last_peered":"2024-08-17T17:24:57.075381+0000","last_clean":"2024-08-17T17:24:57.075381+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:24:57.075381+0000","last_undegraded":"2024-08-17T17:24:57.075381+0000","last_fullsized":"2024-08-17T17:24:57.075381+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":147,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.263158+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:24:43.263158+0000","last_peered":"2024-08-17T17:24:43.263158+0000","last_clean":"2024-08-17T17:24:43.263158+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:24:43.263158+0000","last_undegraded":"2024-08-17T17:24:43.263158+0000","last_fullsized":"2024-08-17T17:24:43.263158+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"158'14823","reported_seq":15998,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:25:00.893531+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:25:00.893531+0000","last_peered":"2024-08-17T17:25:00.893531+0000","last_clean":"2024-08-17T17:25:00.893531+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:25:00.893531+0000","last_undegraded":"2024-08-17T17:25:00.893531+0000","last_fullsized":"2024-08-17T17:25:00.893531+0000","mapping_epoch":135,"log_start":"118'6700","ondisk_log_start":"118'6700","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8123,"log_dups_size":0,"ondisk_log_size":8123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104267776,"num_objects":1591,"num_object_clones":0,"num_object_copies":4773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14823,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":147,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.281008+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:24:43.281008+0000","last_peered":"2024-08-17T17:24:43.281008+0000","last_clean":"2024-08-17T17:24:43.281008+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:24:43.281008+0000","last_undegraded":"2024-08-17T17:24:43.281008+0000","last_fullsized":"2024-08-17T17:24:43.281008+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":36,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:45.908453+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:24:45.908453+0000","last_peered":"2024-08-17T17:24:45.908453+0000","last_clean":"2024-08-17T17:24:45.908453+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:24:45.908453+0000","last_undegraded":"2024-08-17T17:24:45.908453+0000","last_fullsized":"2024-08-17T17:24:45.908453+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"158'14811","reported_seq":14918,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.777115+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:24:57.777115+0000","last_peered":"2024-08-17T17:24:57.777115+0000","last_clean":"2024-08-17T17:24:57.777115+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:24:57.777115+0000","last_undegraded":"2024-08-17T17:24:57.777115+0000","last_fullsized":"2024-08-17T17:24:57.777115+0000","mapping_epoch":111,"log_start":"116'4800","ondisk_log_start":"116'4800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10011,"log_dups_size":0,"ondisk_log_size":10011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110821376,"num_objects":1691,"num_object_clones":0,"num_object_copies":5073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14811,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":148,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.281024+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:24:43.281024+0000","last_peered":"2024-08-17T17:24:43.281024+0000","last_clean":"2024-08-17T17:24:43.281024+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:24:43.281024+0000","last_undegraded":"2024-08-17T17:24:43.281024+0000","last_fullsized":"2024-08-17T17:24:43.281024+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278651+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:24:43.278651+0000","last_peered":"2024-08-17T17:24:43.278651+0000","last_clean":"2024-08-17T17:24:43.278651+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:24:43.278651+0000","last_undegraded":"2024-08-17T17:24:43.278651+0000","last_fullsized":"2024-08-17T17:24:43.278651+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"158'14666","reported_seq":14778,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:57.816919+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:24:57.816919+0000","last_peered":"2024-08-17T17:24:57.816919+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:24:57.816919+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"116'4600","ondisk_log_start":"116'4600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107216896,"num_objects":1636,"num_object_clones":0,"num_object_copies":4908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14666,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-5342},{"shards":"0,2,5","objects":1636}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.280781+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:24:43.280781+0000","last_peered":"2024-08-17T17:24:43.280781+0000","last_clean":"2024-08-17T17:24:43.280781+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:24:43.280781+0000","last_undegraded":"2024-08-17T17:24:43.280781+0000","last_fullsized":"2024-08-17T17:24:43.280781+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.263366+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:24:43.263366+0000","last_peered":"2024-08-17T17:24:43.263366+0000","last_clean":"2024-08-17T17:24:43.263366+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:24:43.263366+0000","last_undegraded":"2024-08-17T17:24:43.263366+0000","last_fullsized":"2024-08-17T17:24:43.263366+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"158'14720","reported_seq":15975,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.719943+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:24:57.719943+0000","last_peered":"2024-08-17T17:24:57.719943+0000","last_clean":"2024-08-17T17:24:57.719943+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:24:57.719943+0000","last_undegraded":"2024-08-17T17:24:57.719943+0000","last_fullsized":"2024-08-17T17:24:57.719943+0000","mapping_epoch":130,"log_start":"116'4700","ondisk_log_start":"116'4700","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14720,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":120,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.074544+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:24:57.074544+0000","last_peered":"2024-08-17T17:24:57.074544+0000","last_clean":"2024-08-17T17:24:57.074544+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:24:57.074544+0000","last_undegraded":"2024-08-17T17:24:57.074544+0000","last_fullsized":"2024-08-17T17:24:57.074544+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.086245+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:24:57.086245+0000","last_peered":"2024-08-17T17:24:57.086245+0000","last_clean":"2024-08-17T17:24:57.086245+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:24:57.086245+0000","last_undegraded":"2024-08-17T17:24:57.086245+0000","last_fullsized":"2024-08-17T17:24:57.086245+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"158'14802","reported_seq":14914,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:57.825646+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:24:57.825646+0000","last_peered":"2024-08-17T17:24:57.825646+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:24:57.825646+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"116'4800","ondisk_log_start":"116'4800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836056,"num_objects":1738,"num_object_clones":0,"num_object_copies":5214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1738,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14802,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1738},{"shards":"2,7","objects":-5376}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":191,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.074269+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:24:57.074269+0000","last_peered":"2024-08-17T17:24:57.074269+0000","last_clean":"2024-08-17T17:24:57.074269+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:24:57.074269+0000","last_undegraded":"2024-08-17T17:24:57.074269+0000","last_fullsized":"2024-08-17T17:24:57.074269+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278881+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:24:43.278881+0000","last_peered":"2024-08-17T17:24:43.278881+0000","last_clean":"2024-08-17T17:24:43.278881+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:24:43.278881+0000","last_undegraded":"2024-08-17T17:24:43.278881+0000","last_fullsized":"2024-08-17T17:24:43.278881+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"158'14896","reported_seq":15082,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:58.834907+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:24:58.834907+0000","last_peered":"2024-08-17T17:24:58.834907+0000","last_clean":"2024-08-17T17:24:58.834907+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:24:58.834907+0000","last_undegraded":"2024-08-17T17:24:58.834907+0000","last_fullsized":"2024-08-17T17:24:58.834907+0000","mapping_epoch":128,"log_start":"119'7856","ondisk_log_start":"119'7856","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115474432,"num_objects":1762,"num_object_clones":0,"num_object_copies":5286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14896,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.263436+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:24:43.263436+0000","last_peered":"2024-08-17T17:24:43.263436+0000","last_clean":"2024-08-17T17:24:43.263436+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:24:43.263436+0000","last_undegraded":"2024-08-17T17:24:43.263436+0000","last_fullsized":"2024-08-17T17:24:43.263436+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279265+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:24:43.279265+0000","last_peered":"2024-08-17T17:24:43.279265+0000","last_clean":"2024-08-17T17:24:43.279265+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:24:43.279265+0000","last_undegraded":"2024-08-17T17:24:43.279265+0000","last_fullsized":"2024-08-17T17:24:43.279265+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"158'15052","reported_seq":21847,"reported_epoch":158,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:01.354283+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:25:01.354283+0000","last_peered":"2024-08-17T17:25:01.354283+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:25:01.354283+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"120'8300","ondisk_log_start":"120'8300","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106692608,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2775,"num_objects_misplaced":1627,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15052,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3351,"num_bytes_recovered":184418304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1627},{"shards":"4,6","objects":-2774}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278537+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:24:43.278537+0000","last_peered":"2024-08-17T17:24:43.278537+0000","last_clean":"2024-08-17T17:24:43.278537+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:24:43.278537+0000","last_undegraded":"2024-08-17T17:24:43.278537+0000","last_fullsized":"2024-08-17T17:24:43.278537+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279149+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:24:43.279149+0000","last_peered":"2024-08-17T17:24:43.279149+0000","last_clean":"2024-08-17T17:24:43.279149+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:24:43.279149+0000","last_undegraded":"2024-08-17T17:24:43.279149+0000","last_fullsized":"2024-08-17T17:24:43.279149+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"158'15217","reported_seq":15373,"reported_epoch":158,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:01.258578+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:25:01.258578+0000","last_peered":"2024-08-17T17:25:01.258578+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:25:01.258578+0000","last_undegraded":"2024-08-17T17:25:01.258578+0000","last_fullsized":"2024-08-17T17:25:01.258578+0000","mapping_epoch":135,"log_start":"125'8541","ondisk_log_start":"125'8541","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102957056,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1571,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15217,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":1571}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":171,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.081580+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:24:57.081580+0000","last_peered":"2024-08-17T17:24:57.081580+0000","last_clean":"2024-08-17T17:24:57.081580+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:24:57.081580+0000","last_undegraded":"2024-08-17T17:24:57.081580+0000","last_fullsized":"2024-08-17T17:24:57.081580+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":158,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.089089+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:24:57.089089+0000","last_peered":"2024-08-17T17:24:57.089089+0000","last_clean":"2024-08-17T17:24:57.089089+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:24:57.089089+0000","last_undegraded":"2024-08-17T17:24:57.089089+0000","last_fullsized":"2024-08-17T17:24:57.089089+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"158'14846","reported_seq":27466,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:25:00.883158+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:25:00.883158+0000","last_peered":"2024-08-17T17:25:00.883158+0000","last_clean":"2024-08-17T17:25:00.883158+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:25:00.883158+0000","last_undegraded":"2024-08-17T17:25:00.883158+0000","last_fullsized":"2024-08-17T17:25:00.883158+0000","mapping_epoch":147,"log_start":"118'6720","ondisk_log_start":"118'6720","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14846,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":452,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.280599+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:24:43.280599+0000","last_peered":"2024-08-17T17:24:43.280599+0000","last_clean":"2024-08-17T17:24:43.280599+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:24:43.280599+0000","last_undegraded":"2024-08-17T17:24:43.280599+0000","last_fullsized":"2024-08-17T17:24:43.280599+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":687,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.085369+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:24:57.085369+0000","last_peered":"2024-08-17T17:24:57.085369+0000","last_clean":"2024-08-17T17:24:57.085369+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:24:57.085369+0000","last_undegraded":"2024-08-17T17:24:57.085369+0000","last_fullsized":"2024-08-17T17:24:57.085369+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.079590+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:24:57.079590+0000","last_peered":"2024-08-17T17:24:57.079590+0000","last_clean":"2024-08-17T17:24:57.079590+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:24:57.079590+0000","last_undegraded":"2024-08-17T17:24:57.079590+0000","last_fullsized":"2024-08-17T17:24:57.079590+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.263982+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:24:43.263982+0000","last_peered":"2024-08-17T17:24:43.263982+0000","last_clean":"2024-08-17T17:24:43.263982+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:24:43.263982+0000","last_undegraded":"2024-08-17T17:24:43.263982+0000","last_fullsized":"2024-08-17T17:24:43.263982+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"158'14815","reported_seq":14902,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:58.812071+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:24:58.812071+0000","last_peered":"2024-08-17T17:24:58.812071+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:24:58.812071+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"119'7756","ondisk_log_start":"119'7756","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7059,"log_dups_size":0,"ondisk_log_size":7059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113180672,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14815,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-5383},{"shards":"0,1,5","objects":1727}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":530,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.280486+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:24:43.280486+0000","last_peered":"2024-08-17T17:24:43.280486+0000","last_clean":"2024-08-17T17:24:43.280486+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:24:43.280486+0000","last_undegraded":"2024-08-17T17:24:43.280486+0000","last_fullsized":"2024-08-17T17:24:43.280486+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":176,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.079283+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:24:57.079283+0000","last_peered":"2024-08-17T17:24:57.079283+0000","last_clean":"2024-08-17T17:24:57.079283+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:24:57.079283+0000","last_undegraded":"2024-08-17T17:24:57.079283+0000","last_fullsized":"2024-08-17T17:24:57.079283+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080300+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:24:57.080300+0000","last_peered":"2024-08-17T17:24:57.080300+0000","last_clean":"2024-08-17T17:24:57.080300+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:24:57.080300+0000","last_undegraded":"2024-08-17T17:24:57.080300+0000","last_fullsized":"2024-08-17T17:24:57.080300+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"158'14641","reported_seq":14751,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:57.778287+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:24:57.778287+0000","last_peered":"2024-08-17T17:24:57.778287+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:24:57.778287+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"116'4600","ondisk_log_start":"116'4600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10041,"log_dups_size":0,"ondisk_log_size":10041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111476736,"num_objects":1701,"num_object_clones":0,"num_object_copies":5103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14641,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1701},{"shards":"2,7","objects":-5372}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":573,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.082522+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:24:57.082522+0000","last_peered":"2024-08-17T17:24:57.082522+0000","last_clean":"2024-08-17T17:24:57.082522+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:24:57.082522+0000","last_undegraded":"2024-08-17T17:24:57.082522+0000","last_fullsized":"2024-08-17T17:24:57.082522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":153,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.063566+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:24:57.063566+0000","last_peered":"2024-08-17T17:24:57.063566+0000","last_clean":"2024-08-17T17:24:57.063566+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:24:57.063566+0000","last_undegraded":"2024-08-17T17:24:57.063566+0000","last_fullsized":"2024-08-17T17:24:57.063566+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":194,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.070471+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:24:57.070471+0000","last_peered":"2024-08-17T17:24:57.070471+0000","last_clean":"2024-08-17T17:24:57.070471+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:24:57.070471+0000","last_undegraded":"2024-08-17T17:24:57.070471+0000","last_fullsized":"2024-08-17T17:24:57.070471+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"158'14541","reported_seq":15697,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:58.278872+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:24:58.278872+0000","last_peered":"2024-08-17T17:24:58.278872+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:24:58.278872+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"118'7171","ondisk_log_start":"118'7171","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132096,"num_objects":1711,"num_object_clones":0,"num_object_copies":5133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14541,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1711}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":972,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:57.056315+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:24:57.056315+0000","last_peered":"2024-08-17T17:24:57.056315+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:24:57.056315+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278887+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:24:43.278887+0000","last_peered":"2024-08-17T17:24:43.278887+0000","last_clean":"2024-08-17T17:24:43.278887+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:24:43.278887+0000","last_undegraded":"2024-08-17T17:24:43.278887+0000","last_fullsized":"2024-08-17T17:24:43.278887+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":59,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.058102+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:24:57.058102+0000","last_peered":"2024-08-17T17:24:57.058102+0000","last_clean":"2024-08-17T17:24:57.058102+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:24:57.058102+0000","last_undegraded":"2024-08-17T17:24:57.058102+0000","last_fullsized":"2024-08-17T17:24:57.058102+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"158'14786","reported_seq":27598,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:58.232299+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:24:58.232299+0000","last_peered":"2024-08-17T17:24:58.232299+0000","last_clean":"2024-08-17T17:24:58.232299+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:24:58.232299+0000","last_undegraded":"2024-08-17T17:24:58.232299+0000","last_fullsized":"2024-08-17T17:24:58.232299+0000","mapping_epoch":147,"log_start":"119'7433","ondisk_log_start":"119'7433","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14786,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.074934+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:24:57.074934+0000","last_peered":"2024-08-17T17:24:57.074934+0000","last_clean":"2024-08-17T17:24:57.074934+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:24:57.074934+0000","last_undegraded":"2024-08-17T17:24:57.074934+0000","last_fullsized":"2024-08-17T17:24:57.074934+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":457,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.058358+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:24:57.058358+0000","last_peered":"2024-08-17T17:24:57.058358+0000","last_clean":"2024-08-17T17:24:57.058358+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:24:57.058358+0000","last_undegraded":"2024-08-17T17:24:57.058358+0000","last_fullsized":"2024-08-17T17:24:57.058358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":163,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.056644+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:24:57.056644+0000","last_peered":"2024-08-17T17:24:57.056644+0000","last_clean":"2024-08-17T17:24:57.056644+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:24:57.056644+0000","last_undegraded":"2024-08-17T17:24:57.056644+0000","last_fullsized":"2024-08-17T17:24:57.056644+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"158'14625","reported_seq":14776,"reported_epoch":158,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:00.715421+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:25:00.715421+0000","last_peered":"2024-08-17T17:25:00.715421+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:25:00.715421+0000","last_undegraded":"2024-08-17T17:25:00.715421+0000","last_fullsized":"2024-08-17T17:25:00.715421+0000","mapping_epoch":135,"log_start":"118'6488","ondisk_log_start":"118'6488","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8137,"log_dups_size":0,"ondisk_log_size":8137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1638,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14625,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":1645}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":160,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.058277+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:24:57.058277+0000","last_peered":"2024-08-17T17:24:57.058277+0000","last_clean":"2024-08-17T17:24:57.058277+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:24:57.058277+0000","last_undegraded":"2024-08-17T17:24:57.058277+0000","last_fullsized":"2024-08-17T17:24:57.058277+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":684,"reported_epoch":157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:43.280915+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:24:43.280915+0000","last_peered":"2024-08-17T17:24:43.280915+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:24:43.280915+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.085279+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:24:57.085279+0000","last_peered":"2024-08-17T17:24:57.085279+0000","last_clean":"2024-08-17T17:24:57.085279+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:24:57.085279+0000","last_undegraded":"2024-08-17T17:24:57.085279+0000","last_fullsized":"2024-08-17T17:24:57.085279+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"158'15024","reported_seq":15202,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:25:01.341555+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:25:01.341555+0000","last_peered":"2024-08-17T17:25:01.341555+0000","last_clean":"2024-08-17T17:25:01.341555+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:25:01.341555+0000","last_undegraded":"2024-08-17T17:25:01.341555+0000","last_fullsized":"2024-08-17T17:25:01.341555+0000","mapping_epoch":128,"log_start":"124'8275","ondisk_log_start":"124'8275","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96468992,"num_objects":1472,"num_object_clones":0,"num_object_copies":4416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15024,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":169,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.075559+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:24:57.075559+0000","last_peered":"2024-08-17T17:24:57.075559+0000","last_clean":"2024-08-17T17:24:57.075559+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:24:57.075559+0000","last_undegraded":"2024-08-17T17:24:57.075559+0000","last_fullsized":"2024-08-17T17:24:57.075559+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":652,"reported_epoch":158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:57.057212+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:24:57.057212+0000","last_peered":"2024-08-17T17:24:57.057212+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:24:57.057212+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":167,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.081585+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:24:57.081585+0000","last_peered":"2024-08-17T17:24:57.081585+0000","last_clean":"2024-08-17T17:24:57.081585+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:24:57.081585+0000","last_undegraded":"2024-08-17T17:24:57.081585+0000","last_fullsized":"2024-08-17T17:24:57.081585+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"158'14900","reported_seq":16224,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:58.651141+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:24:58.651141+0000","last_peered":"2024-08-17T17:24:58.651141+0000","last_clean":"2024-08-17T17:24:58.651141+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:24:58.651141+0000","last_undegraded":"2024-08-17T17:24:58.651141+0000","last_fullsized":"2024-08-17T17:24:58.651141+0000","mapping_epoch":139,"log_start":"118'7330","ondisk_log_start":"118'7330","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113246208,"num_objects":1728,"num_object_clones":0,"num_object_copies":5184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14900,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":188,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.079923+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:24:57.079923+0000","last_peered":"2024-08-17T17:24:57.079923+0000","last_clean":"2024-08-17T17:24:57.079923+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:24:57.079923+0000","last_undegraded":"2024-08-17T17:24:57.079923+0000","last_fullsized":"2024-08-17T17:24:57.079923+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":401,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.264206+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:24:43.264206+0000","last_peered":"2024-08-17T17:24:43.264206+0000","last_clean":"2024-08-17T17:24:43.264206+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:24:43.264206+0000","last_undegraded":"2024-08-17T17:24:43.264206+0000","last_fullsized":"2024-08-17T17:24:43.264206+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278773+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:24:43.278773+0000","last_peered":"2024-08-17T17:24:43.278773+0000","last_clean":"2024-08-17T17:24:43.278773+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:24:43.278773+0000","last_undegraded":"2024-08-17T17:24:43.278773+0000","last_fullsized":"2024-08-17T17:24:43.278773+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278954+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:24:43.278954+0000","last_peered":"2024-08-17T17:24:43.278954+0000","last_clean":"2024-08-17T17:24:43.278954+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:24:43.278954+0000","last_undegraded":"2024-08-17T17:24:43.278954+0000","last_fullsized":"2024-08-17T17:24:43.278954+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":52,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278799+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:24:43.278799+0000","last_peered":"2024-08-17T17:24:43.278799+0000","last_clean":"2024-08-17T17:24:43.278799+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:24:43.278799+0000","last_undegraded":"2024-08-17T17:24:43.278799+0000","last_fullsized":"2024-08-17T17:24:43.278799+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.056527+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:24:57.056527+0000","last_peered":"2024-08-17T17:24:57.056527+0000","last_clean":"2024-08-17T17:24:57.056527+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:24:57.056527+0000","last_undegraded":"2024-08-17T17:24:57.056527+0000","last_fullsized":"2024-08-17T17:24:57.056527+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":153,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.925246+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:24:55.925246+0000","last_peered":"2024-08-17T17:24:55.925246+0000","last_clean":"2024-08-17T17:24:55.925246+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:24:55.925246+0000","last_undegraded":"2024-08-17T17:24:55.925246+0000","last_fullsized":"2024-08-17T17:24:55.925246+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":87,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.098207+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:24:57.098207+0000","last_peered":"2024-08-17T17:24:57.098207+0000","last_clean":"2024-08-17T17:24:57.098207+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:24:57.098207+0000","last_undegraded":"2024-08-17T17:24:57.098207+0000","last_fullsized":"2024-08-17T17:24:57.098207+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.073462+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:24:57.073462+0000","last_peered":"2024-08-17T17:24:57.073462+0000","last_clean":"2024-08-17T17:24:57.073462+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:24:57.073462+0000","last_undegraded":"2024-08-17T17:24:57.073462+0000","last_fullsized":"2024-08-17T17:24:57.073462+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":169,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080602+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:24:57.080602+0000","last_peered":"2024-08-17T17:24:57.080602+0000","last_clean":"2024-08-17T17:24:57.080602+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:24:57.080602+0000","last_undegraded":"2024-08-17T17:24:57.080602+0000","last_fullsized":"2024-08-17T17:24:57.080602+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":170,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.077076+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:24:57.077076+0000","last_peered":"2024-08-17T17:24:57.077076+0000","last_clean":"2024-08-17T17:24:57.077076+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:24:57.077076+0000","last_undegraded":"2024-08-17T17:24:57.077076+0000","last_fullsized":"2024-08-17T17:24:57.077076+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":129,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.264147+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:24:43.264147+0000","last_peered":"2024-08-17T17:24:43.264147+0000","last_clean":"2024-08-17T17:24:43.264147+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:24:43.264147+0000","last_undegraded":"2024-08-17T17:24:43.264147+0000","last_fullsized":"2024-08-17T17:24:43.264147+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":186,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080695+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:24:57.080695+0000","last_peered":"2024-08-17T17:24:57.080695+0000","last_clean":"2024-08-17T17:24:57.080695+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:24:57.080695+0000","last_undegraded":"2024-08-17T17:24:57.080695+0000","last_fullsized":"2024-08-17T17:24:57.080695+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":162,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080219+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:24:57.080219+0000","last_peered":"2024-08-17T17:24:57.080219+0000","last_clean":"2024-08-17T17:24:57.080219+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:24:57.080219+0000","last_undegraded":"2024-08-17T17:24:57.080219+0000","last_fullsized":"2024-08-17T17:24:57.080219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":166,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.056152+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:24:57.056152+0000","last_peered":"2024-08-17T17:24:57.056152+0000","last_clean":"2024-08-17T17:24:57.056152+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:24:57.056152+0000","last_undegraded":"2024-08-17T17:24:57.056152+0000","last_fullsized":"2024-08-17T17:24:57.056152+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":183,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.074748+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:24:57.074748+0000","last_peered":"2024-08-17T17:24:57.074748+0000","last_clean":"2024-08-17T17:24:57.074748+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:24:57.074748+0000","last_undegraded":"2024-08-17T17:24:57.074748+0000","last_fullsized":"2024-08-17T17:24:57.074748+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:51.320484+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:24:51.320484+0000","last_peered":"2024-08-17T17:24:51.320484+0000","last_clean":"2024-08-17T17:24:51.320484+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:24:51.320484+0000","last_undegraded":"2024-08-17T17:24:51.320484+0000","last_fullsized":"2024-08-17T17:24:51.320484+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.082806+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:24:57.082806+0000","last_peered":"2024-08-17T17:24:57.082806+0000","last_clean":"2024-08-17T17:24:57.082806+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:24:57.082806+0000","last_undegraded":"2024-08-17T17:24:57.082806+0000","last_fullsized":"2024-08-17T17:24:57.082806+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":169,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.085269+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:24:57.085269+0000","last_peered":"2024-08-17T17:24:57.085269+0000","last_clean":"2024-08-17T17:24:57.085269+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:24:57.085269+0000","last_undegraded":"2024-08-17T17:24:57.085269+0000","last_fullsized":"2024-08-17T17:24:57.085269+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:51.320509+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:24:51.320509+0000","last_peered":"2024-08-17T17:24:51.320509+0000","last_clean":"2024-08-17T17:24:51.320509+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:24:51.320509+0000","last_undegraded":"2024-08-17T17:24:51.320509+0000","last_fullsized":"2024-08-17T17:24:51.320509+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":142,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.278716+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:24:43.278716+0000","last_peered":"2024-08-17T17:24:43.278716+0000","last_clean":"2024-08-17T17:24:43.278716+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:24:43.278716+0000","last_undegraded":"2024-08-17T17:24:43.278716+0000","last_fullsized":"2024-08-17T17:24:43.278716+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:51.320470+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:24:51.320470+0000","last_peered":"2024-08-17T17:24:51.320470+0000","last_clean":"2024-08-17T17:24:51.320470+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:24:51.320470+0000","last_undegraded":"2024-08-17T17:24:51.320470+0000","last_fullsized":"2024-08-17T17:24:51.320470+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":127,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:43.279094+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:24:43.279094+0000","last_peered":"2024-08-17T17:24:43.279094+0000","last_clean":"2024-08-17T17:24:43.279094+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:24:43.279094+0000","last_undegraded":"2024-08-17T17:24:43.279094+0000","last_fullsized":"2024-08-17T17:24:43.279094+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.073994+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:24:57.073994+0000","last_peered":"2024-08-17T17:24:57.073994+0000","last_clean":"2024-08-17T17:24:57.073994+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:24:57.073994+0000","last_undegraded":"2024-08-17T17:24:57.073994+0000","last_fullsized":"2024-08-17T17:24:57.073994+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":177,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.075274+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:24:57.075274+0000","last_peered":"2024-08-17T17:24:57.075274+0000","last_clean":"2024-08-17T17:24:57.075274+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:24:57.075274+0000","last_undegraded":"2024-08-17T17:24:57.075274+0000","last_fullsized":"2024-08-17T17:24:57.075274+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":42,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:24:46.315982+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:24:46.315982+0000","last_peered":"2024-08-17T17:24:46.315982+0000","last_clean":"2024-08-17T17:24:46.315982+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:24:46.315982+0000","last_undegraded":"2024-08-17T17:24:46.315982+0000","last_fullsized":"2024-08-17T17:24:46.315982+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":179,"reported_epoch":158,"state":"active+clean","last_fresh":"2024-08-17T17:24:57.080502+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:24:57.080502+0000","last_peered":"2024-08-17T17:24:57.080502+0000","last_clean":"2024-08-17T17:24:57.080502+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:24:57.080502+0000","last_undegraded":"2024-08-17T17:24:57.080502+0000","last_fullsized":"2024-08-17T17:24:57.080502+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1750990872,"num_objects":26719,"num_object_clones":0,"num_object_copies":80157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35888,"num_objects_misplaced":4836,"num_objects_unfound":0,"num_objects_dirty":26719,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":237165,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18360,"num_bytes_recovered":1037631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7019180032,"data_stored":7019167816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131095,"ondisk_log_size":131095,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201538,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":644156,"kb_used_data":518644,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":93100996,"statfs":{"total":95995035648,"available":95335419904,"internally_reserved":0,"allocated":531091456,"data_stored":530177429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":128448971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463176,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":536688,"kb_used_data":429288,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":93208464,"statfs":{"total":95995035648,"available":95445467136,"internally_reserved":0,"allocated":439590912,"data_stored":438689339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":109967815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724812,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986448,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1039408,"kb_used_data":837672,"kb_used_omap":1,"kb_used_meta":201726,"kb_avail":92705744,"statfs":{"total":95995035648,"available":94930681856,"internally_reserved":0,"allocated":857776128,"data_stored":856873759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":206567878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435339,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556068,"kb_used_data":443864,"kb_used_omap":5,"kb_used_meta":112186,"kb_avail":93189084,"statfs":{"total":95995035648,"available":95425622016,"internally_reserved":0,"allocated":454516736,"data_stored":453626021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":114878537},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598865,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846248,"kb_used_data":666012,"kb_used_omap":7,"kb_used_meta":180216,"kb_avail":92898904,"statfs":{"total":95995035648,"available":95128477696,"internally_reserved":0,"allocated":681996288,"data_stored":681101957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":184541357},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,8,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762392,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678552,"kb_used_data":560784,"kb_used_omap":7,"kb_used_meta":117752,"kb_avail":93066600,"statfs":{"total":95995035648,"available":95300198400,"internally_reserved":0,"allocated":574242816,"data_stored":573359879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":120578465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925918,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1218984,"kb_used_data":1010848,"kb_used_omap":7,"kb_used_meta":207992,"kb_avail":92526168,"statfs":{"total":95995035648,"available":94746796032,"internally_reserved":0,"allocated":1035108352,"data_stored":1034210706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":212984412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1012858880,"data_stored":1012858880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":560005120,"data_stored":560005120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":673452032,"data_stored":673447960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":442957824,"data_stored":442957824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":847642624,"data_stored":847642624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2537492480,"data_stored":2537488408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":428015616,"data_stored":428015616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":516755456,"data_stored":516751384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:25:05.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:25:05.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:05.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:05.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:05.343 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:05 smithi175 ceph-mon[101434]: from='client.54658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:05.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:05 smithi064 ceph-mon[104516]: from='client.54658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:05 smithi064 ceph-mon[109246]: from='client.54658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:05.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:05.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:05.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:05.799 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:05.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.925 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:05.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.983 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:06.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:06.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:06.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:06.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:06.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:06.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:06.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:06 smithi175 ceph-mon[101434]: pgmap v481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 35918/74067 objects degraded (48.494%); 4836/74067 objects misplaced (6.529%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:06 smithi064 ceph-mon[104516]: pgmap v481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 35918/74067 objects degraded (48.494%); 4836/74067 objects misplaced (6.529%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:06.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:06 smithi064 ceph-mon[109246]: pgmap v481: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.5 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 35918/74067 objects degraded (48.494%); 4836/74067 objects misplaced (6.529%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:06.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:06.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:06.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:06.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:06.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:07.063 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:07.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:07.206 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:07.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:07.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:07.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:07.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:07.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:07.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:07.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:07.742 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:07.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:07.816 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:07.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:07.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:08.005 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:08.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:08.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:08.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:08.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:08 smithi064 ceph-mon[104516]: pgmap v482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35994/70677 objects degraded (50.927%); 4242/70677 objects misplaced (6.002%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:08.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:08 smithi064 ceph-mon[109246]: pgmap v482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35994/70677 objects degraded (50.927%); 4242/70677 objects misplaced (6.002%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:08.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:08.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:08.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:08.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:08 smithi175 ceph-mon[101434]: pgmap v482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 35994/70677 objects degraded (50.927%); 4242/70677 objects misplaced (6.002%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:08.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:08.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:08.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:08.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:08.716 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:08.818 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:08.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:08.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:08.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:09.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:09.173 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:25:09.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:09.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:09.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:09.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:09.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:09.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35994/70677 objects degraded (50.927%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:09.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1232742289' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35994/70677 objects degraded (50.927%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:09.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1232742289' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:09 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.489 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:09.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:09.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:09.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:09 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35994/70677 objects degraded (50.927%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1232742289' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:09 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:09.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:09.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:25:09.747 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:09.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:09.781 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:25:09.781 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:09.823 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:25:09.824 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:09.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:25:09.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:09.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:09.888 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:25:09.888 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:09.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:09.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:09.920 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:25:09.920 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:09.954 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:25:09.954 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:09.988 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:25:09.988 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:10.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:10.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:10.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:10.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:10.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: pgmap v483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 35994/66225 objects degraded (54.351%); 4242/66225 objects misplaced (6.405%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: osdmap e159: 8 total, 8 up, 7 in 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1047244967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: pgmap v483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 35994/66225 objects degraded (54.351%); 4242/66225 objects misplaced (6.405%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: osdmap e159: 8 total, 8 up, 7 in 2024-08-17T17:25:10.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1047244967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:10.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:10.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:10.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:10.568 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:10.601 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: pgmap v483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.4 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 35994/66225 objects degraded (54.351%); 4242/66225 objects misplaced (6.405%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: osdmap e159: 8 total, 8 up, 7 in 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1047244967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:10.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:10.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:10.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:10.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:10.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.028 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.131 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.160 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:11.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.425 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:11.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:11.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:11.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:11.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:11.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:11.711 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:11.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:12.083 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:12.115 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:12.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:12.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:12.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:12.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:12.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:12 smithi064 ceph-mon[104516]: pgmap v485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 35994/64380 objects degraded (55.909%); 4023/64380 objects misplaced (6.249%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:12 smithi064 ceph-mon[109246]: pgmap v485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 35994/64380 objects degraded (55.909%); 4023/64380 objects misplaced (6.249%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:12.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:12.532 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:12.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:12.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:12 smithi175 ceph-mon[101434]: pgmap v485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 35994/64380 objects degraded (55.909%); 4023/64380 objects misplaced (6.249%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:25:12.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:12.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:12.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:12.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:12.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:13.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:13.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:13.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:13.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:13.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:13.320 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35994/64380 objects degraded (55.909%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:13.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35994/64380 objects degraded (55.909%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:13.352 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.634 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35994/64380 objects degraded (55.909%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:13.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:13.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:14.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:14.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:14.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:14.185 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:13] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:25:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.198 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:14.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:14.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:25:14.203 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:25:14.204 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:25:14.204 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:25:14.204 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:25:14.204 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:25:14.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:14.359 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:14.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:14 smithi064 ceph-mon[104516]: pgmap v486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36093/61377 objects degraded (58.805%); 3626/61377 objects misplaced (5.908%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:14 smithi064 ceph-mon[109246]: pgmap v486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36093/61377 objects degraded (58.805%); 3626/61377 objects misplaced (5.908%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:14.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:14.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:14.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:14.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:14.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:14 smithi175 ceph-mon[101434]: pgmap v486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.3 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36093/61377 objects degraded (58.805%); 3626/61377 objects misplaced (5.908%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:14.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:14.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:14.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:14.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:14.880 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:14.919 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:14.919 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 36093/61377 objects degraded (58.805%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":698,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":159,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915134,"num_in_osds":7,"osd_in_since":1723915354,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":78},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":20459,"data_bytes":1343690749,"bytes_used":4589645824,"bytes_avail":667375603712,"bytes_total":671965249536,"degraded_objects":36093,"degraded_total":61377,"degraded_ratio":0.58805415709467712,"misplaced_objects":3626,"misplaced_total":61377,"misplaced_ratio":0.059077504602701329,"recovering_objects_per_sec":24,"recovering_bytes_per_sec":1604834,"recovering_keys_per_sec":0,"num_objects_recovered":245,"num_bytes_recovered":16056320,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":628},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":23,"modified":"2024-08-17T17:19:54.695374+0000","services":{}},"progress_events":{}} 2024-08-17T17:25:14.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:15.008 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:15.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:15.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:15.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:15.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:15.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:15.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/551304919' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:15.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/551304919' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:15.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:15.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:15.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/551304919' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:15.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:15.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:15.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:15.821 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:25:15.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:25:15.840 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:15.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:15.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:25:15.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:15.978 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:16.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:16.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:16.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:16.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:16.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:16.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:16.380 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:16.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:16.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:16 smithi064 ceph-mon[104516]: pgmap v487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 36093/55161 objects degraded (65.432%); 3626/55161 objects misplaced (6.573%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:16.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:16 smithi064 ceph-mon[109246]: pgmap v487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 36093/55161 objects degraded (65.432%); 3626/55161 objects misplaced (6.573%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:16.508 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:16.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:16.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:16.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:16 smithi175 ceph-mon[101434]: pgmap v487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 36093/55161 objects degraded (65.432%); 3626/55161 objects misplaced (6.573%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:16.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:16.772 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:16.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:16.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:16.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:17.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:17.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.076 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:17.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:17.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:17.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:17.319 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:17.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:17.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:17.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:17.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:17.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:17.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:17.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:17.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:17.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:17.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:18.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:18.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:18.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:18.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:18.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:18.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:18.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:18.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:18.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:18.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:18.626 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:18.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:18 smithi175 ceph-mon[101434]: pgmap v488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36174/51699 objects degraded (69.970%); 3046/51699 objects misplaced (5.892%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36174/51699 objects degraded (69.970%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:25:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[104516]: pgmap v488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36174/51699 objects degraded (69.970%); 3046/51699 objects misplaced (5.892%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:18.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36174/51699 objects degraded (69.970%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:18.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:25:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[109246]: pgmap v488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 1.1 GiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36174/51699 objects degraded (69.970%); 3046/51699 objects misplaced (5.892%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36174/51699 objects degraded (69.970%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:25:18.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:18.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:18.794 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:18.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:18.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:18.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:19.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.061 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:19.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:19.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.206 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:19.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:19.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:19.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:19.412 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:25:19.412 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:19.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:25:19.443 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:19.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:19.482 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:25:19.482 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:19.513 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:25:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:19.547 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:25:19.548 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:19.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:19.579 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:25:19.579 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:19.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:19.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:25:19.620 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:19.648 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:25:19.649 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:19.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:19.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:19.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:19.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:19.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:20.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:20.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:20.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:20.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:20.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:20.566 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:20 smithi175 ceph-mon[101434]: pgmap v489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 979 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 36174/46869 objects degraded (77.181%); 3046/46869 objects misplaced (6.499%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:20 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1217610194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:20.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:20.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:20 smithi064 ceph-mon[104516]: pgmap v489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 979 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 36174/46869 objects degraded (77.181%); 3046/46869 objects misplaced (6.499%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:20.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:20 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1217610194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:20.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:20 smithi064 ceph-mon[109246]: pgmap v489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 979 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 36174/46869 objects degraded (77.181%); 3046/46869 objects misplaced (6.499%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:20.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:20 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1217610194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:20.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:20.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:20.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:21.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:21.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:21.108 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:21.140 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:21.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:21.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:21.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.430 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:21.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:21.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:21.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:21.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:21.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:21.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:21.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.983 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:21.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:22.088 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:22.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:22.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:22.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:22.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:22 smithi064 ceph-mon[104516]: pgmap v490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 942 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 36174/45069 objects degraded (80.264%); 2861/45069 objects misplaced (6.348%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:25:22.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:22 smithi064 ceph-mon[109246]: pgmap v490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 942 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 36174/45069 objects degraded (80.264%); 2861/45069 objects misplaced (6.348%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:25:22.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:22.399 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:22.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:22.539 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:22.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:22 smithi175 ceph-mon[101434]: pgmap v490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 942 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 36174/45069 objects degraded (80.264%); 2861/45069 objects misplaced (6.348%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:25:22.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:22.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:22.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:22.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:23.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:23.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:23.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:23.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:23.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:23.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:23.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:23.508 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36174/45069 objects degraded (80.264%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:23.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36174/45069 objects degraded (80.264%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:23.535 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36174/45069 objects degraded (80.264%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:23.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:23.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:23.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:23.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:23.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:23.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:24.017 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:24.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:24.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:24.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:24.234 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:23] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:25:24.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:24.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:24.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:24.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.475 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:24.576 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:24.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:24.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[104516]: pgmap v491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 880 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36278/42087 objects degraded (86.198%); 2456/42087 objects misplaced (5.836%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:25:24.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[109246]: pgmap v491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 880 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36278/42087 objects degraded (86.198%); 2456/42087 objects misplaced (5.836%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:25:24.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:24 smithi175 ceph-mon[101434]: pgmap v491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 880 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 36278/42087 objects degraded (86.198%); 2456/42087 objects misplaced (5.836%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:25:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:24.712 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:24.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:24.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.869 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:24.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:24.992 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:25.094 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:25.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:25.148 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:25.148 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:25.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.249 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:25:25.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:25.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:25.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:25.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:25.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:25.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:25.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:25.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:25.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:25.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:25.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:25.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.832 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:25.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:25.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:25.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:26.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:26.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:26.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:26.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:26.356 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:26.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:26.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[104516]: pgmap v492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 756 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 36278/36174 objects degraded (100.287%); 2456/36174 objects misplaced (6.789%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[104516]: osdmap e160: 8 total, 8 up, 7 in 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[109246]: pgmap v492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 756 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 36278/36174 objects degraded (100.287%); 2456/36174 objects misplaced (6.789%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:25:26.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:26.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:26.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:26 smithi064 ceph-mon[109246]: osdmap e160: 8 total, 8 up, 7 in 2024-08-17T17:25:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:26 smithi175 ceph-mon[101434]: pgmap v492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 756 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 36278/36174 objects degraded (100.287%); 2456/36174 objects misplaced (6.789%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:25:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:26 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:26 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2166948810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:26 smithi175 ceph-mon[101434]: osdmap e160: 8 total, 8 up, 7 in 2024-08-17T17:25:26.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:26.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:26.699 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:26.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:26.797 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:26.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:27.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:27.065 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:27.067 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:25:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:27.186 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:27.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:27.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:27.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:27.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:27.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:27.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:27.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:27.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:27.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:25:27.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:27.786 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:27.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:27.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:27.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:28.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:28.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:28.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:28.099 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:28.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:28.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:28.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[104516]: pgmap v494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 682 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36372/32583 objects degraded (111.629%); 1817/32583 objects misplaced (5.577%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[104516]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36372/32583 objects degraded (111.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[109246]: pgmap v494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 682 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36372/32583 objects degraded (111.629%); 1817/32583 objects misplaced (5.577%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[109246]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36372/32583 objects degraded (111.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:28.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:28.529 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:28.548 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:28 smithi175 ceph-mon[101434]: pgmap v494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 682 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36372/32583 objects degraded (111.629%); 1817/32583 objects misplaced (5.577%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:28.549 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:28 smithi175 ceph-mon[101434]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:28.549 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36372/32583 objects degraded (111.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:28.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:28.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:28.778 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:28.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:28.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:28.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:29.050 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:29.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:29.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:29.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:29.305 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:29.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:29.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:29.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:29.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:29.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:29.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:29.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:29.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:29.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:29.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:29.876 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:29.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:29.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:29.999 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:30.000 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:30.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:30.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:30.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:30.162 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:30.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:30.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:30.288 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:30.390 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:30.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:30.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:30.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:30 smithi175 ceph-mon[101434]: pgmap v495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 583 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 36372/27831 objects degraded (130.689%); 1817/27831 objects misplaced (6.529%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:30.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:30 smithi064 ceph-mon[104516]: pgmap v495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 583 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 36372/27831 objects degraded (130.689%); 1817/27831 objects misplaced (6.529%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:30 smithi064 ceph-mon[109246]: pgmap v495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 583 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 36372/27831 objects degraded (130.689%); 1817/27831 objects misplaced (6.529%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:30.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:30.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:25:30.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:25:30.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:30.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:30.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:31.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:31.060 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:31.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.157 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:31.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:31.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:31.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:31.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.547 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":494,"stamp":"2024-08-17T17:25:26.865609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":714676221,"num_objects":10861,"num_object_clones":0,"num_object_copies":32583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36372,"num_objects_misplaced":1817,"num_objects_unfound":0,"num_objects_dirty":10861,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":254174,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19024,"num_bytes_recovered":1082465444,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132363,"ondisk_log_size":132363,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2947896,"kb_used_data":1861176,"kb_used_omap":33,"kb_used_meta":1086366,"kb_avail":653268168,"statfs":{"total":671965249536,"available":668946604032,"internally_reserved":0,"allocated":1905844224,"data_stored":1899489904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1112439371},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417595392,"num_objects":-6372,"num_object_clones":0,"num_object_copies":-19116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":198,"num_objects_misplaced":-1229,"num_objects_unfound":0,"num_objects_dirty":-6372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6372,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":14745600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003303"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":172,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.191623+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:25:09.191623+0000","last_peered":"2024-08-17T17:25:09.191623+0000","last_clean":"2024-08-17T17:25:09.191623+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:25:09.191623+0000","last_undegraded":"2024-08-17T17:25:09.191623+0000","last_fullsized":"2024-08-17T17:25:09.191623+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":154,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201777+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:25:09.201777+0000","last_peered":"2024-08-17T17:25:09.201777+0000","last_clean":"2024-08-17T17:25:09.201777+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:25:09.201777+0000","last_undegraded":"2024-08-17T17:25:09.201777+0000","last_fullsized":"2024-08-17T17:25:09.201777+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201299+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:25:09.201299+0000","last_peered":"2024-08-17T17:25:09.201299+0000","last_clean":"2024-08-17T17:25:09.201299+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:25:09.201299+0000","last_undegraded":"2024-08-17T17:25:09.201299+0000","last_fullsized":"2024-08-17T17:25:09.201299+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.189963+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:25:09.189963+0000","last_peered":"2024-08-17T17:25:09.189963+0000","last_clean":"2024-08-17T17:25:09.189963+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:25:09.189963+0000","last_undegraded":"2024-08-17T17:25:09.189963+0000","last_fullsized":"2024-08-17T17:25:09.189963+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188502+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:25:09.188502+0000","last_peered":"2024-08-17T17:25:09.188502+0000","last_clean":"2024-08-17T17:25:09.188502+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:25:09.188502+0000","last_undegraded":"2024-08-17T17:25:09.188502+0000","last_fullsized":"2024-08-17T17:25:09.188502+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201671+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:25:09.201671+0000","last_peered":"2024-08-17T17:25:09.201671+0000","last_clean":"2024-08-17T17:25:09.201671+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:25:09.201671+0000","last_undegraded":"2024-08-17T17:25:09.201671+0000","last_fullsized":"2024-08-17T17:25:09.201671+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":172,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192244+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:25:09.192244+0000","last_peered":"2024-08-17T17:25:09.192244+0000","last_clean":"2024-08-17T17:25:09.192244+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:25:09.192244+0000","last_undegraded":"2024-08-17T17:25:09.192244+0000","last_fullsized":"2024-08-17T17:25:09.192244+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":149,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.191991+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:25:09.191991+0000","last_peered":"2024-08-17T17:25:09.191991+0000","last_clean":"2024-08-17T17:25:09.191991+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:25:09.191991+0000","last_undegraded":"2024-08-17T17:25:09.191991+0000","last_fullsized":"2024-08-17T17:25:09.191991+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"160'15776","reported_seq":16953,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.947957+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:25:25.947957+0000","last_peered":"2024-08-17T17:25:25.947957+0000","last_clean":"2024-08-17T17:25:25.947957+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:25:25.947957+0000","last_undegraded":"2024-08-17T17:25:25.947957+0000","last_fullsized":"2024-08-17T17:25:25.947957+0000","mapping_epoch":135,"log_start":"119'7600","ondisk_log_start":"119'7600","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41811968,"num_objects":638,"num_object_clones":0,"num_object_copies":1914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15776,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":149,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201535+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:25:09.201535+0000","last_peered":"2024-08-17T17:25:09.201535+0000","last_clean":"2024-08-17T17:25:09.201535+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:25:09.201535+0000","last_undegraded":"2024-08-17T17:25:09.201535+0000","last_fullsized":"2024-08-17T17:25:09.201535+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201129+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:25:09.201129+0000","last_peered":"2024-08-17T17:25:09.201129+0000","last_clean":"2024-08-17T17:25:09.201129+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:25:09.201129+0000","last_undegraded":"2024-08-17T17:25:09.201129+0000","last_fullsized":"2024-08-17T17:25:09.201129+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"159'15797","reported_seq":15906,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:22.784286+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:25:22.784286+0000","last_peered":"2024-08-17T17:25:22.784286+0000","last_clean":"2024-08-17T17:25:22.784286+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:25:22.784286+0000","last_undegraded":"2024-08-17T17:25:22.784286+0000","last_fullsized":"2024-08-17T17:25:22.784286+0000","mapping_epoch":111,"log_start":"116'5700","ondisk_log_start":"116'5700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10097,"log_dups_size":0,"ondisk_log_size":10097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46202880,"num_objects":705,"num_object_clones":0,"num_object_copies":2115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15797,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":150,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201949+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:25:09.201949+0000","last_peered":"2024-08-17T17:25:09.201949+0000","last_clean":"2024-08-17T17:25:09.201949+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:25:09.201949+0000","last_undegraded":"2024-08-17T17:25:09.201949+0000","last_fullsized":"2024-08-17T17:25:09.201949+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184456+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:25:09.184456+0000","last_peered":"2024-08-17T17:25:09.184456+0000","last_clean":"2024-08-17T17:25:09.184456+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:25:09.184456+0000","last_undegraded":"2024-08-17T17:25:09.184456+0000","last_fullsized":"2024-08-17T17:25:09.184456+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"159'15558","reported_seq":15672,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:22.897393+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:25:22.897393+0000","last_peered":"2024-08-17T17:25:22.897393+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:25:22.897393+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"116'5500","ondisk_log_start":"116'5500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48758784,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15558,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-6234},{"shards":"0,2,5","objects":744}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":121,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201556+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:25:09.201556+0000","last_peered":"2024-08-17T17:25:09.201556+0000","last_clean":"2024-08-17T17:25:09.201556+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:25:09.201556+0000","last_undegraded":"2024-08-17T17:25:09.201556+0000","last_fullsized":"2024-08-17T17:25:09.201556+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:15.741018+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:25:15.741018+0000","last_peered":"2024-08-17T17:25:15.741018+0000","last_clean":"2024-08-17T17:25:15.741018+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:25:15.741018+0000","last_undegraded":"2024-08-17T17:25:15.741018+0000","last_fullsized":"2024-08-17T17:25:15.741018+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"159'15728","reported_seq":16985,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:22.948022+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:25:22.948022+0000","last_peered":"2024-08-17T17:25:22.948022+0000","last_clean":"2024-08-17T17:25:22.948022+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:25:22.948022+0000","last_undegraded":"2024-08-17T17:25:22.948022+0000","last_fullsized":"2024-08-17T17:25:22.948022+0000","mapping_epoch":130,"log_start":"117'5700","ondisk_log_start":"117'5700","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10028,"log_dups_size":0,"ondisk_log_size":10028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48889856,"num_objects":746,"num_object_clones":0,"num_object_copies":2238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15728,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.191772+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:25:09.191772+0000","last_peered":"2024-08-17T17:25:09.191772+0000","last_clean":"2024-08-17T17:25:09.191772+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:25:09.191772+0000","last_undegraded":"2024-08-17T17:25:09.191772+0000","last_fullsized":"2024-08-17T17:25:09.191772+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":178,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.190415+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:25:09.190415+0000","last_peered":"2024-08-17T17:25:09.190415+0000","last_clean":"2024-08-17T17:25:09.190415+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:25:09.190415+0000","last_undegraded":"2024-08-17T17:25:09.190415+0000","last_fullsized":"2024-08-17T17:25:09.190415+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"159'15807","reported_seq":15920,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:22.941850+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:25:22.941850+0000","last_peered":"2024-08-17T17:25:22.941850+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:25:22.941850+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"117'5800","ondisk_log_start":"117'5800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47972376,"num_objects":733,"num_object_clones":0,"num_object_copies":2199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":733,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15807,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":733},{"shards":"2,7","objects":-6381}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":193,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.193309+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:25:09.193309+0000","last_peered":"2024-08-17T17:25:09.193309+0000","last_clean":"2024-08-17T17:25:09.193309+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:25:09.193309+0000","last_undegraded":"2024-08-17T17:25:09.193309+0000","last_fullsized":"2024-08-17T17:25:09.193309+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184850+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:25:09.184850+0000","last_peered":"2024-08-17T17:25:09.184850+0000","last_clean":"2024-08-17T17:25:09.184850+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:25:09.184850+0000","last_undegraded":"2024-08-17T17:25:09.184850+0000","last_fullsized":"2024-08-17T17:25:09.184850+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"159'15932","reported_seq":16120,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:23.759229+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:25:23.759229+0000","last_peered":"2024-08-17T17:25:23.759229+0000","last_clean":"2024-08-17T17:25:23.759229+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:25:23.759229+0000","last_undegraded":"2024-08-17T17:25:23.759229+0000","last_fullsized":"2024-08-17T17:25:23.759229+0000","mapping_epoch":128,"log_start":"126'8856","ondisk_log_start":"126'8856","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47579136,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15932,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":141,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:15.741036+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:25:15.741036+0000","last_peered":"2024-08-17T17:25:15.741036+0000","last_clean":"2024-08-17T17:25:15.741036+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:25:15.741036+0000","last_undegraded":"2024-08-17T17:25:15.741036+0000","last_fullsized":"2024-08-17T17:25:15.741036+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.185032+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:25:09.185032+0000","last_peered":"2024-08-17T17:25:09.185032+0000","last_clean":"2024-08-17T17:25:09.185032+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:25:09.185032+0000","last_undegraded":"2024-08-17T17:25:09.185032+0000","last_fullsized":"2024-08-17T17:25:09.185032+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"160'16095","reported_seq":24078,"reported_epoch":160,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:26.416677+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:25:26.416677+0000","last_peered":"2024-08-17T17:25:26.416677+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:25:26.416677+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"133'9400","ondisk_log_start":"133'9400","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6695,"log_dups_size":0,"ondisk_log_size":6695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38338560,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3229,"num_objects_misplaced":580,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16095,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3944,"num_bytes_recovered":223281152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":580},{"shards":"4,6","objects":-3224}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184378+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:25:09.184378+0000","last_peered":"2024-08-17T17:25:09.184378+0000","last_clean":"2024-08-17T17:25:09.184378+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:25:09.184378+0000","last_undegraded":"2024-08-17T17:25:09.184378+0000","last_fullsized":"2024-08-17T17:25:09.184378+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.185867+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:25:09.185867+0000","last_peered":"2024-08-17T17:25:09.185867+0000","last_clean":"2024-08-17T17:25:09.185867+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:25:09.185867+0000","last_undegraded":"2024-08-17T17:25:09.185867+0000","last_fullsized":"2024-08-17T17:25:09.185867+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"160'16192","reported_seq":16350,"reported_epoch":160,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:26.423808+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:25:26.423808+0000","last_peered":"2024-08-17T17:25:26.423808+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:25:26.423808+0000","last_undegraded":"2024-08-17T17:25:26.423808+0000","last_fullsized":"2024-08-17T17:25:26.423808+0000","mapping_epoch":135,"log_start":"133'9441","ondisk_log_start":"133'9441","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39059456,"num_objects":596,"num_object_clones":0,"num_object_copies":1788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":596,"num_objects_unfound":0,"num_objects_dirty":596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16192,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":596}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":174,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.222121+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:25:09.222121+0000","last_peered":"2024-08-17T17:25:09.222121+0000","last_clean":"2024-08-17T17:25:09.222121+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:25:09.222121+0000","last_undegraded":"2024-08-17T17:25:09.222121+0000","last_fullsized":"2024-08-17T17:25:09.222121+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":161,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.200153+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:25:09.200153+0000","last_peered":"2024-08-17T17:25:09.200153+0000","last_clean":"2024-08-17T17:25:09.200153+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:25:09.200153+0000","last_undegraded":"2024-08-17T17:25:09.200153+0000","last_fullsized":"2024-08-17T17:25:09.200153+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"160'15843","reported_seq":28465,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.940981+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:25:25.940981+0000","last_peered":"2024-08-17T17:25:25.940981+0000","last_clean":"2024-08-17T17:25:25.940981+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:25:25.940981+0000","last_undegraded":"2024-08-17T17:25:25.940981+0000","last_fullsized":"2024-08-17T17:25:25.940981+0000","mapping_epoch":147,"log_start":"119'7720","ondisk_log_start":"119'7720","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8123,"log_dups_size":0,"ondisk_log_size":8123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40173568,"num_objects":613,"num_object_clones":0,"num_object_copies":1839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15843,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":454,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201211+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:25:09.201211+0000","last_peered":"2024-08-17T17:25:09.201211+0000","last_clean":"2024-08-17T17:25:09.201211+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:25:09.201211+0000","last_undegraded":"2024-08-17T17:25:09.201211+0000","last_fullsized":"2024-08-17T17:25:09.201211+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":689,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.203120+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:25:09.203120+0000","last_peered":"2024-08-17T17:25:09.203120+0000","last_clean":"2024-08-17T17:25:09.203120+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:25:09.203120+0000","last_undegraded":"2024-08-17T17:25:09.203120+0000","last_fullsized":"2024-08-17T17:25:09.203120+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192772+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:25:09.192772+0000","last_peered":"2024-08-17T17:25:09.192772+0000","last_clean":"2024-08-17T17:25:09.192772+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:25:09.192772+0000","last_undegraded":"2024-08-17T17:25:09.192772+0000","last_fullsized":"2024-08-17T17:25:09.192772+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192258+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:25:09.192258+0000","last_peered":"2024-08-17T17:25:09.192258+0000","last_clean":"2024-08-17T17:25:09.192258+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:25:09.192258+0000","last_undegraded":"2024-08-17T17:25:09.192258+0000","last_fullsized":"2024-08-17T17:25:09.192258+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"159'15854","reported_seq":15943,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:23.904599+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:25:23.904599+0000","last_peered":"2024-08-17T17:25:23.904599+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:25:23.904599+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"129'8856","ondisk_log_start":"129'8856","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45088768,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15854,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-6422},{"shards":"0,1,5","objects":688}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":532,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201148+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:25:09.201148+0000","last_peered":"2024-08-17T17:25:09.201148+0000","last_clean":"2024-08-17T17:25:09.201148+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:25:09.201148+0000","last_undegraded":"2024-08-17T17:25:09.201148+0000","last_fullsized":"2024-08-17T17:25:09.201148+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":178,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.185170+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:25:09.185170+0000","last_peered":"2024-08-17T17:25:09.185170+0000","last_clean":"2024-08-17T17:25:09.185170+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:25:09.185170+0000","last_undegraded":"2024-08-17T17:25:09.185170+0000","last_fullsized":"2024-08-17T17:25:09.185170+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188628+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:25:09.188628+0000","last_peered":"2024-08-17T17:25:09.188628+0000","last_clean":"2024-08-17T17:25:09.188628+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:25:09.188628+0000","last_undegraded":"2024-08-17T17:25:09.188628+0000","last_fullsized":"2024-08-17T17:25:09.188628+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"159'15622","reported_seq":15733,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:22.948702+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:25:22.948702+0000","last_peered":"2024-08-17T17:25:22.948702+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:25:22.948702+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"116'5600","ondisk_log_start":"116'5600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15622,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":720},{"shards":"2,7","objects":-6353}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":575,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.195275+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:25:09.195275+0000","last_peered":"2024-08-17T17:25:09.195275+0000","last_clean":"2024-08-17T17:25:09.195275+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:25:09.195275+0000","last_undegraded":"2024-08-17T17:25:09.195275+0000","last_fullsized":"2024-08-17T17:25:09.195275+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":155,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.174242+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:25:09.174242+0000","last_peered":"2024-08-17T17:25:09.174242+0000","last_clean":"2024-08-17T17:25:09.174242+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:25:09.174242+0000","last_undegraded":"2024-08-17T17:25:09.174242+0000","last_fullsized":"2024-08-17T17:25:09.174242+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":196,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.185793+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:25:09.185793+0000","last_peered":"2024-08-17T17:25:09.185793+0000","last_clean":"2024-08-17T17:25:09.185793+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:25:09.185793+0000","last_undegraded":"2024-08-17T17:25:09.185793+0000","last_fullsized":"2024-08-17T17:25:09.185793+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"159'15516","reported_seq":16674,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:23.335813+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:25:23.335813+0000","last_peered":"2024-08-17T17:25:23.335813+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:25:23.335813+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"124'8171","ondisk_log_start":"124'8171","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48234496,"num_objects":736,"num_object_clones":0,"num_object_copies":2208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15516,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":736}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":974,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:09.178488+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:25:09.178488+0000","last_peered":"2024-08-17T17:25:09.178488+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:25:09.178488+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184815+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:25:09.184815+0000","last_peered":"2024-08-17T17:25:09.184815+0000","last_clean":"2024-08-17T17:25:09.184815+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:25:09.184815+0000","last_undegraded":"2024-08-17T17:25:09.184815+0000","last_fullsized":"2024-08-17T17:25:09.184815+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":61,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.179781+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:25:09.179781+0000","last_peered":"2024-08-17T17:25:09.179781+0000","last_clean":"2024-08-17T17:25:09.179781+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:25:09.179781+0000","last_undegraded":"2024-08-17T17:25:09.179781+0000","last_fullsized":"2024-08-17T17:25:09.179781+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"159'15831","reported_seq":28645,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:23.369122+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:25:23.369122+0000","last_peered":"2024-08-17T17:25:23.369122+0000","last_clean":"2024-08-17T17:25:23.369122+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:25:23.369122+0000","last_undegraded":"2024-08-17T17:25:23.369122+0000","last_fullsized":"2024-08-17T17:25:23.369122+0000","mapping_epoch":147,"log_start":"125'8433","ondisk_log_start":"125'8433","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46465024,"num_objects":709,"num_object_clones":0,"num_object_copies":2127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15831,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192692+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:25:09.192692+0000","last_peered":"2024-08-17T17:25:09.192692+0000","last_clean":"2024-08-17T17:25:09.192692+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:25:09.192692+0000","last_undegraded":"2024-08-17T17:25:09.192692+0000","last_fullsized":"2024-08-17T17:25:09.192692+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":459,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.180049+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:25:09.180049+0000","last_peered":"2024-08-17T17:25:09.180049+0000","last_clean":"2024-08-17T17:25:09.180049+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:25:09.180049+0000","last_undegraded":"2024-08-17T17:25:09.180049+0000","last_fullsized":"2024-08-17T17:25:09.180049+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":165,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.179356+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:25:09.179356+0000","last_peered":"2024-08-17T17:25:09.179356+0000","last_clean":"2024-08-17T17:25:09.179356+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:25:09.179356+0000","last_undegraded":"2024-08-17T17:25:09.179356+0000","last_fullsized":"2024-08-17T17:25:09.179356+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"160'15627","reported_seq":15780,"reported_epoch":160,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:25.713353+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:25:25.713353+0000","last_peered":"2024-08-17T17:25:25.713353+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:25:25.713353+0000","last_undegraded":"2024-08-17T17:25:25.713353+0000","last_fullsized":"2024-08-17T17:25:25.713353+0000","mapping_epoch":135,"log_start":"119'7488","ondisk_log_start":"119'7488","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42139648,"num_objects":643,"num_object_clones":0,"num_object_copies":1929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":641,"num_objects_unfound":0,"num_objects_dirty":643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15627,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":643}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":162,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.179967+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:25:09.179967+0000","last_peered":"2024-08-17T17:25:09.179967+0000","last_clean":"2024-08-17T17:25:09.179967+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:25:09.179967+0000","last_undegraded":"2024-08-17T17:25:09.179967+0000","last_fullsized":"2024-08-17T17:25:09.179967+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":687,"reported_epoch":160,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:25.281064+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:25:25.281064+0000","last_peered":"2024-08-17T17:25:25.281064+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:25:25.281064+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":160,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.190338+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:25:09.190338+0000","last_peered":"2024-08-17T17:25:09.190338+0000","last_clean":"2024-08-17T17:25:09.190338+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:25:09.190338+0000","last_undegraded":"2024-08-17T17:25:09.190338+0000","last_fullsized":"2024-08-17T17:25:09.190338+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"160'15935","reported_seq":16115,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:26.361276+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:25:26.361276+0000","last_peered":"2024-08-17T17:25:26.361276+0000","last_clean":"2024-08-17T17:25:26.361276+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:25:26.361276+0000","last_undegraded":"2024-08-17T17:25:26.361276+0000","last_fullsized":"2024-08-17T17:25:26.361276+0000","mapping_epoch":128,"log_start":"131'9175","ondisk_log_start":"131'9175","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36765696,"num_objects":561,"num_object_clones":0,"num_object_copies":1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15935,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":172,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192466+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:25:09.192466+0000","last_peered":"2024-08-17T17:25:09.192466+0000","last_clean":"2024-08-17T17:25:09.192466+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:25:09.192466+0000","last_undegraded":"2024-08-17T17:25:09.192466+0000","last_fullsized":"2024-08-17T17:25:09.192466+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":654,"reported_epoch":159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:09.196287+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:25:09.196287+0000","last_peered":"2024-08-17T17:25:09.196287+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:25:09.196287+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.220594+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:25:09.220594+0000","last_peered":"2024-08-17T17:25:09.220594+0000","last_clean":"2024-08-17T17:25:09.220594+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:25:09.220594+0000","last_undegraded":"2024-08-17T17:25:09.220594+0000","last_fullsized":"2024-08-17T17:25:09.220594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"159'15938","reported_seq":17264,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:23.676257+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:25:23.676257+0000","last_peered":"2024-08-17T17:25:23.676257+0000","last_clean":"2024-08-17T17:25:23.676257+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:25:23.676257+0000","last_undegraded":"2024-08-17T17:25:23.676257+0000","last_fullsized":"2024-08-17T17:25:23.676257+0000","mapping_epoch":139,"log_start":"125'8430","ondisk_log_start":"125'8430","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45219840,"num_objects":690,"num_object_clones":0,"num_object_copies":2070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15938,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":190,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188419+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:25:09.188419+0000","last_peered":"2024-08-17T17:25:09.188419+0000","last_clean":"2024-08-17T17:25:09.188419+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:25:09.188419+0000","last_undegraded":"2024-08-17T17:25:09.188419+0000","last_fullsized":"2024-08-17T17:25:09.188419+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":403,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192257+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:25:09.192257+0000","last_peered":"2024-08-17T17:25:09.192257+0000","last_clean":"2024-08-17T17:25:09.192257+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:25:09.192257+0000","last_undegraded":"2024-08-17T17:25:09.192257+0000","last_fullsized":"2024-08-17T17:25:09.192257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184518+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:25:09.184518+0000","last_peered":"2024-08-17T17:25:09.184518+0000","last_clean":"2024-08-17T17:25:09.184518+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:25:09.184518+0000","last_undegraded":"2024-08-17T17:25:09.184518+0000","last_fullsized":"2024-08-17T17:25:09.184518+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184841+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:25:09.184841+0000","last_peered":"2024-08-17T17:25:09.184841+0000","last_clean":"2024-08-17T17:25:09.184841+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:25:09.184841+0000","last_undegraded":"2024-08-17T17:25:09.184841+0000","last_fullsized":"2024-08-17T17:25:09.184841+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":54,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184765+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:25:09.184765+0000","last_peered":"2024-08-17T17:25:09.184765+0000","last_clean":"2024-08-17T17:25:09.184765+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:25:09.184765+0000","last_undegraded":"2024-08-17T17:25:09.184765+0000","last_fullsized":"2024-08-17T17:25:09.184765+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.179209+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:25:09.179209+0000","last_peered":"2024-08-17T17:25:09.179209+0000","last_clean":"2024-08-17T17:25:09.179209+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:25:09.179209+0000","last_undegraded":"2024-08-17T17:25:09.179209+0000","last_fullsized":"2024-08-17T17:25:09.179209+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":155,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201382+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:25:09.201382+0000","last_peered":"2024-08-17T17:25:09.201382+0000","last_clean":"2024-08-17T17:25:09.201382+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:25:09.201382+0000","last_undegraded":"2024-08-17T17:25:09.201382+0000","last_fullsized":"2024-08-17T17:25:09.201382+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201873+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:25:09.201873+0000","last_peered":"2024-08-17T17:25:09.201873+0000","last_clean":"2024-08-17T17:25:09.201873+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:25:09.201873+0000","last_undegraded":"2024-08-17T17:25:09.201873+0000","last_fullsized":"2024-08-17T17:25:09.201873+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":123,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192066+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:25:09.192066+0000","last_peered":"2024-08-17T17:25:09.192066+0000","last_clean":"2024-08-17T17:25:09.192066+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:25:09.192066+0000","last_undegraded":"2024-08-17T17:25:09.192066+0000","last_fullsized":"2024-08-17T17:25:09.192066+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188899+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:25:09.188899+0000","last_peered":"2024-08-17T17:25:09.188899+0000","last_clean":"2024-08-17T17:25:09.188899+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:25:09.188899+0000","last_undegraded":"2024-08-17T17:25:09.188899+0000","last_fullsized":"2024-08-17T17:25:09.188899+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.219816+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:25:09.219816+0000","last_peered":"2024-08-17T17:25:09.219816+0000","last_clean":"2024-08-17T17:25:09.219816+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:25:09.219816+0000","last_undegraded":"2024-08-17T17:25:09.219816+0000","last_fullsized":"2024-08-17T17:25:09.219816+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192212+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:25:09.192212+0000","last_peered":"2024-08-17T17:25:09.192212+0000","last_clean":"2024-08-17T17:25:09.192212+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:25:09.192212+0000","last_undegraded":"2024-08-17T17:25:09.192212+0000","last_fullsized":"2024-08-17T17:25:09.192212+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.222278+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:25:09.222278+0000","last_peered":"2024-08-17T17:25:09.222278+0000","last_clean":"2024-08-17T17:25:09.222278+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:25:09.222278+0000","last_undegraded":"2024-08-17T17:25:09.222278+0000","last_fullsized":"2024-08-17T17:25:09.222278+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188626+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:25:09.188626+0000","last_peered":"2024-08-17T17:25:09.188626+0000","last_clean":"2024-08-17T17:25:09.188626+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:25:09.188626+0000","last_undegraded":"2024-08-17T17:25:09.188626+0000","last_fullsized":"2024-08-17T17:25:09.188626+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":169,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.178599+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:25:09.178599+0000","last_peered":"2024-08-17T17:25:09.178599+0000","last_clean":"2024-08-17T17:25:09.178599+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:25:09.178599+0000","last_undegraded":"2024-08-17T17:25:09.178599+0000","last_fullsized":"2024-08-17T17:25:09.178599+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":185,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.191626+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:25:09.191626+0000","last_peered":"2024-08-17T17:25:09.191626+0000","last_clean":"2024-08-17T17:25:09.191626+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:25:09.191626+0000","last_undegraded":"2024-08-17T17:25:09.191626+0000","last_fullsized":"2024-08-17T17:25:09.191626+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184570+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:25:09.184570+0000","last_peered":"2024-08-17T17:25:09.184570+0000","last_clean":"2024-08-17T17:25:09.184570+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:25:09.184570+0000","last_undegraded":"2024-08-17T17:25:09.184570+0000","last_fullsized":"2024-08-17T17:25:09.184570+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.195713+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:25:09.195713+0000","last_peered":"2024-08-17T17:25:09.195713+0000","last_clean":"2024-08-17T17:25:09.195713+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:25:09.195713+0000","last_undegraded":"2024-08-17T17:25:09.195713+0000","last_fullsized":"2024-08-17T17:25:09.195713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":171,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.190561+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:25:09.190561+0000","last_peered":"2024-08-17T17:25:09.190561+0000","last_clean":"2024-08-17T17:25:09.190561+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:25:09.190561+0000","last_undegraded":"2024-08-17T17:25:09.190561+0000","last_fullsized":"2024-08-17T17:25:09.190561+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184752+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:25:09.184752+0000","last_peered":"2024-08-17T17:25:09.184752+0000","last_clean":"2024-08-17T17:25:09.184752+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:25:09.184752+0000","last_undegraded":"2024-08-17T17:25:09.184752+0000","last_fullsized":"2024-08-17T17:25:09.184752+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":145,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.185187+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:25:09.185187+0000","last_peered":"2024-08-17T17:25:09.185187+0000","last_clean":"2024-08-17T17:25:09.185187+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:25:09.185187+0000","last_undegraded":"2024-08-17T17:25:09.185187+0000","last_fullsized":"2024-08-17T17:25:09.185187+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184547+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:25:09.184547+0000","last_peered":"2024-08-17T17:25:09.184547+0000","last_clean":"2024-08-17T17:25:09.184547+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:25:09.184547+0000","last_undegraded":"2024-08-17T17:25:09.184547+0000","last_fullsized":"2024-08-17T17:25:09.184547+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":130,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184983+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:25:09.184983+0000","last_peered":"2024-08-17T17:25:09.184983+0000","last_clean":"2024-08-17T17:25:09.184983+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:25:09.184983+0000","last_undegraded":"2024-08-17T17:25:09.184983+0000","last_fullsized":"2024-08-17T17:25:09.184983+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192360+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:25:09.192360+0000","last_peered":"2024-08-17T17:25:09.192360+0000","last_clean":"2024-08-17T17:25:09.192360+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:25:09.192360+0000","last_undegraded":"2024-08-17T17:25:09.192360+0000","last_fullsized":"2024-08-17T17:25:09.192360+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":179,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.192133+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:25:09.192133+0000","last_peered":"2024-08-17T17:25:09.192133+0000","last_clean":"2024-08-17T17:25:09.192133+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:25:09.192133+0000","last_undegraded":"2024-08-17T17:25:09.192133+0000","last_fullsized":"2024-08-17T17:25:09.192133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":44,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.184328+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:25:09.184328+0000","last_peered":"2024-08-17T17:25:09.184328+0000","last_clean":"2024-08-17T17:25:09.184328+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:25:09.184328+0000","last_undegraded":"2024-08-17T17:25:09.184328+0000","last_fullsized":"2024-08-17T17:25:09.184328+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":181,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.188786+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:25:09.188786+0000","last_peered":"2024-08-17T17:25:09.188786+0000","last_clean":"2024-08-17T17:25:09.188786+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:25:09.188786+0000","last_undegraded":"2024-08-17T17:25:09.188786+0000","last_fullsized":"2024-08-17T17:25:09.188786+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":709885976,"num_objects":10833,"num_object_clones":0,"num_object_copies":32499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36342,"num_objects_misplaced":1817,"num_objects_unfound":0,"num_objects_dirty":10833,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":253051,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18953,"num_bytes_recovered":1076494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4225642496,"data_stored":4225630280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131181,"ondisk_log_size":131181,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201543,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342368,"kb_used_data":207060,"kb_used_omap":1,"kb_used_meta":135294,"kb_avail":93402784,"statfs":{"total":95995035648,"available":95644450816,"internally_reserved":0,"allocated":212029440,"data_stored":211099697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":138541515},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463181,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277696,"kb_used_data":170296,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":93467456,"statfs":{"total":95995035648,"available":95710674944,"internally_reserved":0,"allocated":174383104,"data_stored":173473673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":109967815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724817,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986453,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":540820,"kb_used_data":333256,"kb_used_omap":1,"kb_used_meta":207486,"kb_avail":93204332,"statfs":{"total":95995035648,"available":95441235968,"internally_reserved":0,"allocated":341254144,"data_stored":340336059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":212466118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435344,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":314608,"kb_used_data":188520,"kb_used_omap":5,"kb_used_meta":126074,"kb_avail":93430544,"statfs":{"total":95995035648,"available":95672877056,"internally_reserved":0,"allocated":193044480,"data_stored":192145907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":129099849},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598870,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469108,"kb_used_data":288748,"kb_used_omap":7,"kb_used_meta":180216,"kb_avail":93276044,"statfs":{"total":95995035648,"available":95514669056,"internally_reserved":0,"allocated":295677952,"data_stored":294775763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":184541357},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762397,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":356072,"kb_used_data":234144,"kb_used_omap":7,"kb_used_meta":121912,"kb_avail":93389080,"statfs":{"total":95995035648,"available":95630417920,"internally_reserved":0,"allocated":239763456,"data_stored":238872661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":124838305},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925923,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":647224,"kb_used_data":439152,"kb_used_omap":7,"kb_used_meta":207992,"kb_avail":93097928,"statfs":{"total":95995035648,"available":95332278272,"internally_reserved":0,"allocated":449691648,"data_stored":448786144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":212984412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":425721856,"data_stored":425721856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":227409920,"data_stored":227409920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":286855168,"data_stored":286851096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":180944896,"data_stored":180944896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":330629120,"data_stored":330629120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2408189952,"data_stored":2408185880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":164626432,"data_stored":164626432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":201265152,"data_stored":201261080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:25:31.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.651 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:31.667 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:31.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:31.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:31.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:31.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:32.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:32.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:32.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:32.156 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:32.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:32.258 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:32.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:32.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:32.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:32 smithi064 ceph-mon[104516]: pgmap v496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 545 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 36372/26049 objects degraded (139.629%); 1610/26049 objects misplaced (6.181%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:32 smithi064 ceph-mon[109246]: pgmap v496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 545 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 36372/26049 objects degraded (139.629%); 1610/26049 objects misplaced (6.181%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:32.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:32.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:32.582 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:32 smithi175 ceph-mon[101434]: pgmap v496: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 545 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 36372/26049 objects degraded (139.629%); 1610/26049 objects misplaced (6.181%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:25:32.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:32.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:32.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.827 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:32.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:32.885 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:32.885 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:32.928 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:32.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:33.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:33.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:33.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:33.225 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:33.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:33.360 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:33.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:33.507 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:33.569 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36372/26049 objects degraded (139.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:33.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:25:33.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36372/26049 objects degraded (139.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:33.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36372/26049 objects degraded (139.629%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:33.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:33.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:33.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:33.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:33.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:33.898 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:33.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:34.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:34.061 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:34.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:34.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:34.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:34.295 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:33] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-08-17T17:25:34.316 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:34 smithi175 ceph-mon[101434]: pgmap v497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 484 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36433/23109 objects degraded (157.657%); 1197/23109 objects misplaced (5.180%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:34.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:34.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:34.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:25:34.561 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:34.593 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:34.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:34 smithi064 ceph-mon[104516]: pgmap v497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 484 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36433/23109 objects degraded (157.657%); 1197/23109 objects misplaced (5.180%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:34.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:34 smithi064 ceph-mon[109246]: pgmap v497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 484 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 36433/23109 objects degraded (157.657%); 1197/23109 objects misplaced (5.180%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:34.695 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:34.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:34.858 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:34.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:35.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:35.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:35.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:25:35.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:35.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:35.183 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:25:35.183 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:35.234 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:25:35.234 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:35.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:35.269 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:25:35.269 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:35.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:35.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:25:35.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:35.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:25:35.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:35.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:35.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:25:35.392 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:35.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:35.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4025841801' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:35.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:25:35.430 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:35.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:35.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:35.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4025841801' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:35.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4025841801' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:35.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140794 5.e 5:70adc07f:::benchmark_data_smithi064_147470_object8850:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:25:35.744 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.52345196600001, 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.01103878, 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140794 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:25:35.745 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217387+0000", 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.746 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217394+0000", 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8109999999999997e-06 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217408+0000", 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3536999999999999e-05 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.747 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217415+0000", 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5680000000000002e-06 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217613+0000", 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019751099999999999 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217621+0000", 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4849999999999992e-06 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.748 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210300+0000", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.99267841499999998 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210423+0000", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012385 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210935+0000", 2024-08-17T17:25:35.749 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00051183100000000003 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225429+0000", 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014493239999999999 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225526+0000", 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7489000000000001e-05 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.750 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228353+0000", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0028269509999999999 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228388+0000", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5176000000000003e-05 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228428+0000", 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9483999999999997e-05 2024-08-17T17:25:35.751 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140802 5.1 5:8420ad0a:::benchmark_data_smithi064_147470_object8858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.45782993, 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93395603000000005, 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.752 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140802 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.753 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283009+0000", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283017+0000", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6709999999999999e-06 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283032+0000", 2024-08-17T17:25:35.754 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5159e-05 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283063+0000", 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0406e-05 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283108+0000", 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4935999999999998e-05 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.755 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283118+0000", 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0757000000000001e-05 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210288+0000", 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.92716919799999997 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210421+0000", 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013376499999999999 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.756 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210968+0000", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054626499999999999 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216852+0000", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058840200000000002 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216887+0000", 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5522000000000003e-05 2024-08-17T17:25:35.757 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216894+0000", 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.63e-06 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216933+0000", 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9208999999999997e-05 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.758 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216967+0000", 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3924000000000002e-05 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140822 5.1 5:8accaffa:::benchmark_data_smithi064_147470_object8878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.36356121200001, 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85167474700000001, 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.759 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140822 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.760 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377278+0000", 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377285+0000", 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.761 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377301+0000", 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5423000000000001e-05 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377309+0000", 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0390000000000005e-06 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.762 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377380+0000", 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0784999999999994e-05 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377389+0000", 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1179999999999998e-06 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216990+0000", 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83960149699999997 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.763 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217105+0000", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114775 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217195+0000", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9908999999999996e-05 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228788+0000", 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011592966 2024-08-17T17:25:35.764 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228835+0000", 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6978000000000003e-05 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228873+0000", 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7605000000000003e-05 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.765 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228921+0000", 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8013000000000003e-05 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228954+0000", 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3775000000000001e-05 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140827 5.1 5:8b17e64e:::benchmark_data_smithi064_147470_object8883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.363514966, 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85178876800000003, 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140827 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.767 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377325+0000", 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.768 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377328+0000", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1370000000000002e-06 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377335+0000", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8260000000000004e-06 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377340+0000", 2024-08-17T17:25:35.769 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1730000000000003e-06 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377409+0000", 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8702999999999996e-05 2024-08-17T17:25:35.770 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377415+0000", 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8869999999999997e-06 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220644+0000", 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84322923900000002 2024-08-17T17:25:35.771 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220744+0000", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9375000000000003e-05 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220857+0000", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000113417 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228842+0000", 2024-08-17T17:25:35.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079852320000000001 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228994+0000", 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001518 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229024+0000", 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0190999999999999e-05 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.774 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229078+0000", 2024-08-17T17:25:35.774 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3427999999999997e-05 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229115+0000", 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6875999999999997e-05 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140835 5.1 5:88614440:::benchmark_data_smithi064_147470_object8891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:25:35.775 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.35534847700001, 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85468788299999998, 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140835 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:25:35.776 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385491+0000", 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.777 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385500+0000", 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.738e-06 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385518+0000", 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8902e-05 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385527+0000", 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7320000000000004e-06 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.778 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385583+0000", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6035000000000001e-05 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385593+0000", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4089999999999992e-06 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224815+0000", 2024-08-17T17:25:35.779 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83922198699999995 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224935+0000", 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120415 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225062+0000", 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000127467 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.780 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229173+0000", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041108539999999997 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235012+0000", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058387229999999997 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240111+0000", 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0050989340000000003 2024-08-17T17:25:35.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240148+0000", 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6748000000000001e-05 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240180+0000", 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2509999999999999e-05 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.782 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140846 5.1 5:8aef7983:::benchmark_data_smithi064_147470_object8902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.33786491000001, 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84633296700000005, 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140846 2024-08-17T17:25:35.783 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.784 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402975+0000", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402981+0000", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.19e-06 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402993+0000", 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1949e-05 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403001+0000", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403040+0000", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9366000000000002e-05 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403047+0000", 2024-08-17T17:25:35.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3019999999999997e-06 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229200+0000", 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82615223199999999 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229313+0000", 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011332699999999999 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229435+0000", 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012217400000000001 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240217+0000", 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010781689000000001 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240246+0000", 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9513999999999999e-05 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.788 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249221+0000", 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0089744300000000003 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249270+0000", 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9008999999999998e-05 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249309+0000", 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9031e-05 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140847 5.e 5:7ffdcb03:::benchmark_data_smithi064_147470_object8903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.33608774300001, 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82381208399999994, 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.790 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140847 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.791 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404752+0000", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404758+0000", 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7039999999999999e-06 2024-08-17T17:25:35.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404770+0000", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2148e-05 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404777+0000", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6889999999999998e-06 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404822+0000", 2024-08-17T17:25:35.793 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4827000000000003e-05 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404854+0000", 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2366999999999997e-05 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218174+0000", 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81331985799999995 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.794 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218278+0000", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010474799999999999 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218390+0000", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111401 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225460+0000", 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0070701369999999998 2024-08-17T17:25:35.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225563+0000", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010276 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228497+0000", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002934613 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228535+0000", 2024-08-17T17:25:35.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7539000000000001e-05 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228565+0000", 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0238000000000001e-05 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140853 5.1 5:8ec8c268:::benchmark_data_smithi064_147470_object8909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:25:35.797 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.334426493, 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.843158564, 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140853 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.798 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406413+0000", 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406419+0000", 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.057e-06 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406433+0000", 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4328000000000001e-05 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406440+0000", 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8419999999999999e-06 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406489+0000", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9506999999999999e-05 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406498+0000", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3429999999999996e-06 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235037+0000", 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82853932900000005 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.801 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235131+0000", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4499000000000004e-05 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235239+0000", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107617 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249349+0000", 2024-08-17T17:25:35.802 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014109981000000001 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249391+0000", 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1950999999999999e-05 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249499+0000", 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010786899999999999 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249541+0000", 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249573+0000", 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2308999999999998e-05 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140856 5.e 5:7672cac1:::benchmark_data_smithi064_147470_object8912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.804 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.326590791, 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82466934800000002, 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140856 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:25:35.805 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414249+0000", 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.806 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414255+0000", 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414267+0000", 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1759e-05 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414272+0000", 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2039999999999996e-06 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414318+0000", 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6096e-05 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.808 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414326+0000", 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222146+0000", 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.80782070699999997 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222239+0000", 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2288999999999997e-05 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.809 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222337+0000", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8740000000000007e-05 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228879+0000", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0065412170000000002 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.230356+0000", 2024-08-17T17:25:35.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014777900000000001 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238835+0000", 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0084788959999999997 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238881+0000", 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5371000000000003e-05 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238919+0000", 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8642e-05 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140876 5.1 5:8e383c61:::benchmark_data_smithi064_147470_object8932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.28958455, 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79844427600000001, 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.812 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140876 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:25:35.813 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451255+0000", 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451262+0000", 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1709999999999996e-06 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.814 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451275+0000", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2833000000000001e-05 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451293+0000", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8309999999999999e-05 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451336+0000", 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3052e-05 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.815 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451346+0000", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8879999999999992e-06 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243525+0000", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79217860100000004 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243656+0000", 2024-08-17T17:25:35.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013163500000000001 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243790+0000", 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013325699999999999 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249472+0000", 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0056827220000000003 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.817 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249604+0000", 2024-08-17T17:25:35.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000131984 2024-08-17T17:25:35.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.818 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249646+0000", 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2058999999999998e-05 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249675+0000", 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8381000000000002e-05 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249701+0000", 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5769e-05 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140877 5.e 5:70e8904e:::benchmark_data_smithi064_147470_object8933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 194.28736734699999, 2024-08-17T17:25:35.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.78553511300000001, 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140877 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453472+0000", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453480+0000", 2024-08-17T17:25:35.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453490+0000", 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0473e-05 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453497+0000", 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0570000000000004e-06 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453537+0000", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9613000000000002e-05 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453544+0000", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225585+0000", 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.77204120700000001 2024-08-17T17:25:35.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225685+0000", 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000100185 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225791+0000", 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010613800000000001 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229184+0000", 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033933909999999999 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238791+0000", 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096060130000000001 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238950+0000", 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015964399999999999 2024-08-17T17:25:35.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238981+0000", 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1239999999999999e-05 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.239009+0000", 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7203e-05 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150664 5.f 5:f7ffbecc:::benchmark_data_smithi064_147470_object18720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 178.04673335000001, 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24416275700000001, 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150664 2024-08-17T17:25:35.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694106+0000", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694113+0000", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1339999999999998e-06 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694126+0000", 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2644e-05 2024-08-17T17:25:35.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694132+0000", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6150000000000002e-06 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.739879+0000", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.045747115999999997 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.740006+0000", 2024-08-17T17:25:35.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012663900000000001 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.792997+0000", 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.052990644000000003 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793094+0000", 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7040000000000006e-05 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793282+0000", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018835799999999999 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804038+0000", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010755603000000001 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804047+0000", 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9590000000000001e-06 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.934993+0000", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.13094678300000001 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.935051+0000", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8128999999999999e-05 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.938270+0000", 2024-08-17T17:25:35.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032187499999999998 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150743 5.f 5:f8fc72c3:::benchmark_data_smithi064_147470_object18799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 177.928608371, 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27548363100000001, 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.836 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150743 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812231+0000", 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812237+0000", 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.694e-06 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812248+0000", 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1012000000000001e-05 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812253+0000", 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2719999999999997e-06 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824221+0000", 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011967396999999999 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824354+0000", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013296799999999999 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988177+0000", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.163823843 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988257+0000", 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9561000000000006e-05 2024-08-17T17:25:35.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988353+0000", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5855999999999998e-05 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.004347+0000", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015994228999999999 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.012117+0000", 2024-08-17T17:25:35.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0077704050000000002 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083403+0000", 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071285541999999993 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083488+0000", 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5028e-05 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.087716+0000", 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042280720000000003 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:163575 5.f 5:fcf25f79:::benchmark_data_smithi064_147470_object31631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 155.21912482799999, 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25042483300000001, 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.843 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 163575 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521715+0000", 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521720+0000", 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3839999999999996e-06 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521728+0000", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3000000000000002e-06 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521733+0000", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0459999999999996e-06 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.536593+0000", 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014859387 2024-08-17T17:25:35.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.536696+0000", 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010328299999999999 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.649726+0000", 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11302983899999999 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.649845+0000", 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011899 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.650032+0000", 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000187159 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.657391+0000", 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0073592029999999999 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.674848+0000", 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.017456639999999999 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.761792+0000", 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.086943764000000007 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.761865+0000", 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3592999999999998e-05 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.772141+0000", 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01027551 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:165359 5.f 5:faf23924:::benchmark_data_smithi064_147470_object33415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 152.33679001199999, 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27524345, 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.851 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 165359 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:25:35.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404049+0000", 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404055+0000", 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9150000000000001e-06 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404069+0000", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4323e-05 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404076+0000", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8040000000000004e-06 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.415728+0000", 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011651729 2024-08-17T17:25:35.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.415800+0000", 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1552999999999997e-05 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.606816+0000", 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.191016189 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.606935+0000", 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000119521 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.607081+0000", 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014616799999999999 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.627535+0000", 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.020453091999999999 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.636155+0000", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0086204880000000008 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.675139+0000", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.038983775999999998 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.675200+0000", 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1042000000000003e-05 2024-08-17T17:25:35.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.679294+0000", 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040944279999999998 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:167570 5.f 5:fb418e04:::benchmark_data_smithi064_147470_object35626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:25:35.858 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 148.57293859000001, 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25951065200000001, 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167570 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:25:35.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167901+0000", 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167905+0000", 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0060000000000003e-06 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167924+0000", 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8298e-05 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167931+0000", 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8349999999999998e-06 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.182560+0000", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014629647000000001 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.182671+0000", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111246 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338341+0000", 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.15566972000000001 2024-08-17T17:25:35.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338446+0000", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104911 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338605+0000", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015865200000000001 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.346574+0000", 2024-08-17T17:25:35.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079697080000000007 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.346630+0000", 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5788000000000002e-05 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.423542+0000", 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.076911882000000001 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.423605+0000", 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3003000000000006e-05 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.427413+0000", 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038078029999999998 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:176227 5.f 5:f2f7e856:::benchmark_data_smithi064_147470_object44283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:25:35.865 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 133.67143249099999, 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29274401999999999, 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 176227 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:25:35.866 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069407+0000", 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069417+0000", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7990000000000005e-06 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069427+0000", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0095000000000001e-05 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069434+0000", 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7340000000000002e-06 2024-08-17T17:25:35.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.085581+0000", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.016147509000000001 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.085657+0000", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5525999999999995e-05 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.305768+0000", 2024-08-17T17:25:35.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.22011108300000001 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.305919+0000", 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000151674 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.306089+0000", 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016917399999999999 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.315695+0000", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096059730000000003 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.315736+0000", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0970999999999998e-05 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.357779+0000", 2024-08-17T17:25:35.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.042042989000000003 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.357818+0000", 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9650000000000002e-05 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.362152+0000", 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043341930000000001 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:181547 5.f 5:fd5cb98e:::benchmark_data_smithi064_147470_object49603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:31.524667+0000", 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 124.21617421800001, 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.20150100500000001, 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 181547 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524667+0000", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524667+0000", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524666+0000", 2024-08-17T17:25:35.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524679+0000", 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3227999999999999e-05 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524690+0000", 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0688e-05 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.524695+0000", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3709999999999999e-06 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.533101+0000", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.008406373 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.533211+0000", 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010942000000000001 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.647705+0000", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.114493993 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.647802+0000", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6877000000000001e-05 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.647955+0000", 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000153843 2024-08-17T17:25:35.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.662553+0000", 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014597882 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.662647+0000", 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3770000000000005e-05 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.722572+0000", 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.059925452999999997 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.722609+0000", 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6491e-05 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:31.726168+0000", 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035586390000000002 2024-08-17T17:25:35.882 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:196002 5.f 5:f3e23b7f:::benchmark_data_smithi064_147470_object64058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 101.691147575, 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24430998900000001, 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.883 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 196002 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049692+0000", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049703+0000", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1243000000000001e-05 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049715+0000", 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2023e-05 2024-08-17T17:25:35.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049723+0000", 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7090000000000002e-06 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.064549+0000", 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014826295999999999 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.064671+0000", 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121821 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236326+0000", 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.171655102 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236429+0000", 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000103484 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236612+0000", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018228100000000001 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.251683+0000", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015071319 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.254894+0000", 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032105319999999999 2024-08-17T17:25:35.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.290320+0000", 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.035426080999999998 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.290383+0000", 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3212000000000004e-05 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.294003+0000", 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036203960000000001 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:209812 5.f 5:f9f68153:::benchmark_data_smithi064_147470_object77868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 79.732547005000001, 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.22315481500000001, 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:35.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 209812 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:25:35.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008293+0000", 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008299+0000", 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.942e-06 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:25:35.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008310+0000", 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1817999999999999e-05 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008317+0000", 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.029e-06 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.021703+0000", 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013385660000000001 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.021809+0000", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010545800000000001 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203361+0000", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.181552132 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203466+0000", 2024-08-17T17:25:35.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105599 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203617+0000", 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015100700000000001 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.218447+0000", 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014829956 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:35.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.218495+0000", 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7503999999999998e-05 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.229017+0000", 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010522635000000001 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.229054+0000", 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6179e-05 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:25:35.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.231449+0000", 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002395112 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:25:35.897 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:25:35.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:36.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:36.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:36.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:36.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:36.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:36.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:36.332 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:25:36.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:36.457 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:36.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:36.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:36.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:36.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[104516]: pgmap v498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 359 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36433/17082 objects degraded (213.283%); 1197/17082 objects misplaced (7.007%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3409368166' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[109246]: pgmap v498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 359 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36433/17082 objects degraded (213.283%); 1197/17082 objects misplaced (7.007%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3409368166' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:36 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:36 smithi175 ceph-mon[101434]: pgmap v498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 359 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 36433/17082 objects degraded (213.283%); 1197/17082 objects misplaced (7.007%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:25:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3409368166' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:36 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:36.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:36.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:36.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:36.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:37.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:37.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:25:37.061 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:37.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:37.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:37.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.357 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:37 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:37.357 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:37 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:37.358 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:37 smithi175 ceph-mon[101434]: osdmap e161: 8 total, 8 up, 7 in 2024-08-17T17:25:37.394 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[104516]: osdmap e161: 8 total, 8 up, 7 in 2024-08-17T17:25:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:37 smithi064 ceph-mon[109246]: osdmap e161: 8 total, 8 up, 7 in 2024-08-17T17:25:37.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:37.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:37.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:37.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:37.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:37.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:37.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:37.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:38.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.058 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:38.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:38.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:38.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:38.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:38.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:38.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:38.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:38.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:38.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:38 smithi175 ceph-mon[101434]: pgmap v500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 289 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36486/13725 objects degraded (265.836%); 662/13725 objects misplaced (4.823%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36486/13725 objects degraded (265.836%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:38 smithi064 ceph-mon[104516]: pgmap v500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 289 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36486/13725 objects degraded (265.836%); 662/13725 objects misplaced (4.823%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36486/13725 objects degraded (265.836%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:38 smithi064 ceph-mon[109246]: pgmap v500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 289 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 36486/13725 objects degraded (265.836%); 662/13725 objects misplaced (4.823%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36486/13725 objects degraded (265.836%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:38.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:38.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:38.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:38.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:38.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:38.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:39.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:39.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:39.203 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:25:39.203 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:25:39.203 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:25:39.203 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:25:39.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:39.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:39.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:39.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:39.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:39.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:39.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:25:39.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.783 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:39.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:39.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:39.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:39.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:25:40.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:40.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:40.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:40.219 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:40.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:40.278 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:40.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:40.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:40.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:40.439 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:40 smithi175 ceph-mon[101434]: pgmap v501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 196 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 36486/9279 objects degraded (393.210%); 662/9279 objects misplaced (7.134%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:40.440 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:40.440 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:40.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:40.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:40.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:40.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[104516]: pgmap v501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 196 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 36486/9279 objects degraded (393.210%); 662/9279 objects misplaced (7.134%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[109246]: pgmap v501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 196 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 36486/9279 objects degraded (393.210%); 662/9279 objects misplaced (7.134%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:40.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:40.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:40.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:40.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:40.994 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:41.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.023 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:41.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:41.139 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:41.155 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:41.193 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:41.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:41.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:41.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.427 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:41.518 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:41.519 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:25:41.528 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:41.554 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:41.656 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.684 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.765 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.765 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:41.785 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:41.823 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:41.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:41.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:41.973 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 131941 objects 2024-08-17T17:25:41.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :217.071 2024-08-17T17:25:42.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.086 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:42.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:42.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:42.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:42.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:42.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:42.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:42.388 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2024-08-17T17:25:40.871600+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":167385085,"num_objects":2510,"num_object_clones":0,"num_object_copies":7530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36486,"num_objects_misplaced":464,"num_objects_unfound":0,"num_objects_dirty":2510,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":262525,"num_write_kb":8459266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19296,"num_bytes_recovered":1100291236,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132414,"ondisk_log_size":132414,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1642196,"kb_used_data":486232,"kb_used_omap":33,"kb_used_meta":1155486,"kb_avail":654573868,"statfs":{"total":671965249536,"available":670283640832,"internally_reserved":0,"allocated":497901568,"data_stored":491468668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1183218251},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,2,3,6,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-404553728,"num_objects":-6173,"num_object_clones":0,"num_object_copies":-18519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":114,"num_objects_misplaced":-1146,"num_objects_unfound":0,"num_objects_dirty":-6173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6173,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":272,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004642"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":176,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326150+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:25:36.326150+0000","last_peered":"2024-08-17T17:25:36.326150+0000","last_clean":"2024-08-17T17:25:36.326150+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:25:36.326150+0000","last_undegraded":"2024-08-17T17:25:36.326150+0000","last_fullsized":"2024-08-17T17:25:36.326150+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":154,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201777+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:25:09.201777+0000","last_peered":"2024-08-17T17:25:09.201777+0000","last_clean":"2024-08-17T17:25:09.201777+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:25:09.201777+0000","last_undegraded":"2024-08-17T17:25:09.201777+0000","last_fullsized":"2024-08-17T17:25:09.201777+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201299+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:25:09.201299+0000","last_peered":"2024-08-17T17:25:09.201299+0000","last_clean":"2024-08-17T17:25:09.201299+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:25:09.201299+0000","last_undegraded":"2024-08-17T17:25:09.201299+0000","last_fullsized":"2024-08-17T17:25:09.201299+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.353881+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:25:36.353881+0000","last_peered":"2024-08-17T17:25:36.353881+0000","last_clean":"2024-08-17T17:25:36.353881+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:25:36.353881+0000","last_undegraded":"2024-08-17T17:25:36.353881+0000","last_fullsized":"2024-08-17T17:25:36.353881+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354328+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:25:36.354328+0000","last_peered":"2024-08-17T17:25:36.354328+0000","last_clean":"2024-08-17T17:25:36.354328+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:25:36.354328+0000","last_undegraded":"2024-08-17T17:25:36.354328+0000","last_fullsized":"2024-08-17T17:25:36.354328+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201671+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:25:09.201671+0000","last_peered":"2024-08-17T17:25:09.201671+0000","last_clean":"2024-08-17T17:25:09.201671+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:25:09.201671+0000","last_undegraded":"2024-08-17T17:25:09.201671+0000","last_fullsized":"2024-08-17T17:25:09.201671+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326433+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:25:36.326433+0000","last_peered":"2024-08-17T17:25:36.326433+0000","last_clean":"2024-08-17T17:25:36.326433+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:25:36.326433+0000","last_undegraded":"2024-08-17T17:25:36.326433+0000","last_fullsized":"2024-08-17T17:25:36.326433+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":150,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285232+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:25:25.285232+0000","last_peered":"2024-08-17T17:25:25.285232+0000","last_clean":"2024-08-17T17:25:25.285232+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:25:25.285232+0000","last_undegraded":"2024-08-17T17:25:25.285232+0000","last_fullsized":"2024-08-17T17:25:25.285232+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"160'16182","reported_seq":17359,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:35.967353+0000","last_change":"2024-08-17T17:22:36.171617+0000","last_active":"2024-08-17T17:25:35.967353+0000","last_peered":"2024-08-17T17:25:35.967353+0000","last_clean":"2024-08-17T17:25:35.967353+0000","last_became_active":"2024-08-17T17:22:36.153622+0000","last_became_peered":"2024-08-17T17:22:36.153622+0000","last_unstale":"2024-08-17T17:25:35.967353+0000","last_undegraded":"2024-08-17T17:25:35.967353+0000","last_fullsized":"2024-08-17T17:25:35.967353+0000","mapping_epoch":135,"log_start":"119'8000","ondisk_log_start":"119'8000","created":111,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:38.102375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15204352,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16182,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":149,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201535+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:25:09.201535+0000","last_peered":"2024-08-17T17:25:09.201535+0000","last_clean":"2024-08-17T17:25:09.201535+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:25:09.201535+0000","last_undegraded":"2024-08-17T17:25:09.201535+0000","last_fullsized":"2024-08-17T17:25:09.201535+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201129+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:25:09.201129+0000","last_peered":"2024-08-17T17:25:09.201129+0000","last_clean":"2024-08-17T17:25:09.201129+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:25:09.201129+0000","last_undegraded":"2024-08-17T17:25:09.201129+0000","last_fullsized":"2024-08-17T17:25:09.201129+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"161'16370","reported_seq":16483,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:37.976678+0000","last_change":"2024-08-17T17:20:22.803837+0000","last_active":"2024-08-17T17:25:37.976678+0000","last_peered":"2024-08-17T17:25:37.976678+0000","last_clean":"2024-08-17T17:25:37.976678+0000","last_became_active":"2024-08-17T17:20:22.803528+0000","last_became_peered":"2024-08-17T17:20:22.803528+0000","last_unstale":"2024-08-17T17:25:37.976678+0000","last_undegraded":"2024-08-17T17:25:37.976678+0000","last_fullsized":"2024-08-17T17:25:37.976678+0000","mapping_epoch":111,"log_start":"117'6300","ondisk_log_start":"117'6300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:59.112585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8650752,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16370,"num_write_kb":528064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":150,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201949+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:25:09.201949+0000","last_peered":"2024-08-17T17:25:09.201949+0000","last_clean":"2024-08-17T17:25:09.201949+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:25:09.201949+0000","last_undegraded":"2024-08-17T17:25:09.201949+0000","last_fullsized":"2024-08-17T17:25:09.201949+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277554+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:25:25.277554+0000","last_peered":"2024-08-17T17:25:25.277554+0000","last_clean":"2024-08-17T17:25:25.277554+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:25:25.277554+0000","last_undegraded":"2024-08-17T17:25:25.277554+0000","last_fullsized":"2024-08-17T17:25:25.277554+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"161'16170","reported_seq":16288,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:37.958325+0000","last_change":"2024-08-17T17:22:36.547961+0000","last_active":"2024-08-17T17:25:37.958325+0000","last_peered":"2024-08-17T17:25:37.958325+0000","last_clean":"2024-08-17T17:22:34.118026+0000","last_became_active":"2024-08-17T17:22:36.517833+0000","last_became_peered":"2024-08-17T17:22:36.517833+0000","last_unstale":"2024-08-17T17:25:37.958325+0000","last_undegraded":"2024-08-17T17:22:36.263031+0000","last_fullsized":"2024-08-17T17:22:36.141513+0000","mapping_epoch":135,"log_start":"117'6100","ondisk_log_start":"117'6100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8650752,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16170,"num_write_kb":521664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2","objects":-6846},{"shards":"0,2,5","objects":132}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":121,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201556+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:25:09.201556+0000","last_peered":"2024-08-17T17:25:09.201556+0000","last_clean":"2024-08-17T17:25:09.201556+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:25:09.201556+0000","last_undegraded":"2024-08-17T17:25:09.201556+0000","last_fullsized":"2024-08-17T17:25:09.201556+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":123,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285285+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:25:25.285285+0000","last_peered":"2024-08-17T17:25:25.285285+0000","last_clean":"2024-08-17T17:25:25.285285+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:25:25.285285+0000","last_undegraded":"2024-08-17T17:25:25.285285+0000","last_fullsized":"2024-08-17T17:25:25.285285+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"161'16321","reported_seq":17580,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:37.984527+0000","last_change":"2024-08-17T17:22:28.061671+0000","last_active":"2024-08-17T17:25:37.984527+0000","last_peered":"2024-08-17T17:25:37.984527+0000","last_clean":"2024-08-17T17:25:37.984527+0000","last_became_active":"2024-08-17T17:22:28.051117+0000","last_became_peered":"2024-08-17T17:22:28.051117+0000","last_unstale":"2024-08-17T17:25:37.984527+0000","last_undegraded":"2024-08-17T17:25:37.984527+0000","last_fullsized":"2024-08-17T17:25:37.984527+0000","mapping_epoch":130,"log_start":"117'6300","ondisk_log_start":"117'6300","created":111,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10021,"log_dups_size":0,"ondisk_log_size":10021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:55.319986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10027008,"num_objects":153,"num_object_clones":0,"num_object_copies":459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16321,"num_write_kb":527168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326280+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:25:36.326280+0000","last_peered":"2024-08-17T17:25:36.326280+0000","last_clean":"2024-08-17T17:25:36.326280+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:25:36.326280+0000","last_undegraded":"2024-08-17T17:25:36.326280+0000","last_fullsized":"2024-08-17T17:25:36.326280+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":182,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.361952+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:25:36.361952+0000","last_peered":"2024-08-17T17:25:36.361952+0000","last_clean":"2024-08-17T17:25:36.361952+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:25:36.361952+0000","last_undegraded":"2024-08-17T17:25:36.361952+0000","last_fullsized":"2024-08-17T17:25:36.361952+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"161'16394","reported_seq":16512,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:37.963623+0000","last_change":"2024-08-17T17:22:36.579195+0000","last_active":"2024-08-17T17:25:37.963623+0000","last_peered":"2024-08-17T17:25:37.963623+0000","last_clean":"2024-08-17T17:22:34.115724+0000","last_became_active":"2024-08-17T17:22:36.545065+0000","last_became_peered":"2024-08-17T17:22:36.545065+0000","last_unstale":"2024-08-17T17:25:37.963623+0000","last_undegraded":"2024-08-17T17:22:36.268937+0000","last_fullsized":"2024-08-17T17:22:36.140199+0000","mapping_epoch":135,"log_start":"117'6300","ondisk_log_start":"117'6300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9502744,"num_objects":146,"num_object_clones":0,"num_object_copies":438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":146,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16394,"num_write_kb":529217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":146},{"shards":"2,7","objects":-6968}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":198,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328172+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:25:36.328172+0000","last_peered":"2024-08-17T17:25:36.328172+0000","last_clean":"2024-08-17T17:25:36.328172+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:25:36.328172+0000","last_undegraded":"2024-08-17T17:25:36.328172+0000","last_fullsized":"2024-08-17T17:25:36.328172+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.278040+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:25:25.278040+0000","last_peered":"2024-08-17T17:25:25.278040+0000","last_clean":"2024-08-17T17:25:25.278040+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:25:25.278040+0000","last_undegraded":"2024-08-17T17:25:25.278040+0000","last_fullsized":"2024-08-17T17:25:25.278040+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"161'16533","reported_seq":16725,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:38.884564+0000","last_change":"2024-08-17T17:22:22.214669+0000","last_active":"2024-08-17T17:25:38.884564+0000","last_peered":"2024-08-17T17:25:38.884564+0000","last_clean":"2024-08-17T17:25:38.884564+0000","last_became_active":"2024-08-17T17:22:22.202413+0000","last_became_peered":"2024-08-17T17:22:22.202413+0000","last_unstale":"2024-08-17T17:25:38.884564+0000","last_undegraded":"2024-08-17T17:25:38.884564+0000","last_fullsized":"2024-08-17T17:25:38.884564+0000","mapping_epoch":128,"log_start":"139'9556","ondisk_log_start":"139'9556","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:43.311209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8192000,"num_objects":125,"num_object_clones":0,"num_object_copies":375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16533,"num_write_kb":533056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":2162688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285411+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:25:25.285411+0000","last_peered":"2024-08-17T17:25:25.285411+0000","last_clean":"2024-08-17T17:25:25.285411+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:25:25.285411+0000","last_undegraded":"2024-08-17T17:25:25.285411+0000","last_fullsized":"2024-08-17T17:25:25.285411+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.278140+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:25:25.278140+0000","last_peered":"2024-08-17T17:25:25.278140+0000","last_clean":"2024-08-17T17:25:25.278140+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:25:25.278140+0000","last_undegraded":"2024-08-17T17:25:25.278140+0000","last_fullsized":"2024-08-17T17:25:25.278140+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"161'16483","reported_seq":25010,"reported_epoch":161,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:36.485164+0000","last_change":"2024-08-17T17:22:36.644450+0000","last_active":"2024-08-17T17:25:36.485164+0000","last_peered":"2024-08-17T17:25:36.485164+0000","last_clean":"2024-08-17T17:22:34.075081+0000","last_became_active":"2024-08-17T17:22:36.547224+0000","last_became_peered":"2024-08-17T17:22:36.547224+0000","last_unstale":"2024-08-17T17:25:36.485164+0000","last_undegraded":"2024-08-17T17:22:36.204308+0000","last_fullsized":"2024-08-17T17:22:36.141995+0000","mapping_epoch":135,"log_start":"140'9800","ondisk_log_start":"140'9800","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:42.067199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3343,"num_objects_misplaced":194,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16483,"num_write_kb":533760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4216,"num_bytes_recovered":241106944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":194},{"shards":"4,6","objects":-3340}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277475+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:25:25.277475+0000","last_peered":"2024-08-17T17:25:25.277475+0000","last_clean":"2024-08-17T17:25:25.277475+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:25:25.277475+0000","last_undegraded":"2024-08-17T17:25:25.277475+0000","last_fullsized":"2024-08-17T17:25:25.277475+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277922+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:25:25.277922+0000","last_peered":"2024-08-17T17:25:25.277922+0000","last_clean":"2024-08-17T17:25:25.277922+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:25:25.277922+0000","last_undegraded":"2024-08-17T17:25:25.277922+0000","last_fullsized":"2024-08-17T17:25:25.277922+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"161'16566","reported_seq":16725,"reported_epoch":161,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:36.323716+0000","last_change":"2024-08-17T17:22:36.497113+0000","last_active":"2024-08-17T17:25:36.323716+0000","last_peered":"2024-08-17T17:25:36.323716+0000","last_clean":"2024-08-17T17:22:34.139777+0000","last_became_active":"2024-08-17T17:22:36.481354+0000","last_became_peered":"2024-08-17T17:22:36.481354+0000","last_unstale":"2024-08-17T17:25:36.323716+0000","last_undegraded":"2024-08-17T17:25:36.323716+0000","last_fullsized":"2024-08-17T17:25:36.323716+0000","mapping_epoch":135,"log_start":"140'9841","ondisk_log_start":"140'9841","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:42.218968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":222,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16566,"num_write_kb":537216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":222}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":178,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328025+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:25:36.328025+0000","last_peered":"2024-08-17T17:25:36.328025+0000","last_clean":"2024-08-17T17:25:36.328025+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:25:36.328025+0000","last_undegraded":"2024-08-17T17:25:36.328025+0000","last_fullsized":"2024-08-17T17:25:36.328025+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":165,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.335908+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:25:36.335908+0000","last_peered":"2024-08-17T17:25:36.335908+0000","last_clean":"2024-08-17T17:25:36.335908+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:25:36.335908+0000","last_undegraded":"2024-08-17T17:25:36.335908+0000","last_fullsized":"2024-08-17T17:25:36.335908+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"160'16212","reported_seq":28834,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:35.905092+0000","last_change":"2024-08-17T17:23:24.121653+0000","last_active":"2024-08-17T17:25:35.905092+0000","last_peered":"2024-08-17T17:25:35.905092+0000","last_clean":"2024-08-17T17:25:35.905092+0000","last_became_active":"2024-08-17T17:23:24.091385+0000","last_became_peered":"2024-08-17T17:23:24.091385+0000","last_unstale":"2024-08-17T17:25:35.905092+0000","last_undegraded":"2024-08-17T17:25:35.905092+0000","last_fullsized":"2024-08-17T17:25:35.905092+0000","mapping_epoch":147,"log_start":"119'8020","ondisk_log_start":"119'8020","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:03.962971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15990784,"num_objects":244,"num_object_clones":0,"num_object_copies":732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16212,"num_write_kb":526592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6270,"num_bytes_recovered":410910720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":454,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201211+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:25:09.201211+0000","last_peered":"2024-08-17T17:25:09.201211+0000","last_clean":"2024-08-17T17:25:09.201211+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:25:09.201211+0000","last_undegraded":"2024-08-17T17:25:09.201211+0000","last_fullsized":"2024-08-17T17:25:09.201211+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":693,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.334985+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:25:36.334985+0000","last_peered":"2024-08-17T17:25:36.334985+0000","last_clean":"2024-08-17T17:25:36.334985+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:25:36.334985+0000","last_undegraded":"2024-08-17T17:25:36.334985+0000","last_fullsized":"2024-08-17T17:25:36.334985+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326375+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:25:36.326375+0000","last_peered":"2024-08-17T17:25:36.326375+0000","last_clean":"2024-08-17T17:25:36.326375+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:25:36.326375+0000","last_undegraded":"2024-08-17T17:25:36.326375+0000","last_fullsized":"2024-08-17T17:25:36.326375+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":126,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285670+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:25:25.285670+0000","last_peered":"2024-08-17T17:25:25.285670+0000","last_clean":"2024-08-17T17:25:25.285670+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:25:25.285670+0000","last_undegraded":"2024-08-17T17:25:25.285670+0000","last_fullsized":"2024-08-17T17:25:25.285670+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"161'16439","reported_seq":16532,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:38.895371+0000","last_change":"2024-08-17T17:22:36.540552+0000","last_active":"2024-08-17T17:25:38.895371+0000","last_peered":"2024-08-17T17:25:38.895371+0000","last_clean":"2024-08-17T17:22:34.111201+0000","last_became_active":"2024-08-17T17:22:36.514728+0000","last_became_peered":"2024-08-17T17:22:36.514728+0000","last_unstale":"2024-08-17T17:25:38.895371+0000","last_undegraded":"2024-08-17T17:22:36.189609+0000","last_fullsized":"2024-08-17T17:22:36.137739+0000","mapping_epoch":135,"log_start":"137'9456","ondisk_log_start":"137'9456","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6983,"log_dups_size":0,"ondisk_log_size":6983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6750208,"num_objects":103,"num_object_clones":0,"num_object_copies":309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16439,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1","objects":-7007},{"shards":"0,1,5","objects":103}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":532,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201148+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:25:09.201148+0000","last_peered":"2024-08-17T17:25:09.201148+0000","last_clean":"2024-08-17T17:25:09.201148+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:25:09.201148+0000","last_undegraded":"2024-08-17T17:25:09.201148+0000","last_fullsized":"2024-08-17T17:25:09.201148+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.348162+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:25:36.348162+0000","last_peered":"2024-08-17T17:25:36.348162+0000","last_clean":"2024-08-17T17:25:36.348162+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:25:36.348162+0000","last_undegraded":"2024-08-17T17:25:36.348162+0000","last_fullsized":"2024-08-17T17:25:36.348162+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":179,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354473+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:25:36.354473+0000","last_peered":"2024-08-17T17:25:36.354473+0000","last_clean":"2024-08-17T17:25:36.354473+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:25:36.354473+0000","last_undegraded":"2024-08-17T17:25:36.354473+0000","last_fullsized":"2024-08-17T17:25:36.354473+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"161'16183","reported_seq":16298,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:37.974354+0000","last_change":"2024-08-17T17:22:36.546678+0000","last_active":"2024-08-17T17:25:37.974354+0000","last_peered":"2024-08-17T17:25:37.974354+0000","last_clean":"2024-08-17T17:22:34.136251+0000","last_became_active":"2024-08-17T17:22:36.511323+0000","last_became_peered":"2024-08-17T17:22:36.511323+0000","last_unstale":"2024-08-17T17:25:37.974354+0000","last_undegraded":"2024-08-17T17:22:36.226044+0000","last_fullsized":"2024-08-17T17:22:36.140335+0000","mapping_epoch":135,"log_start":"117'6100","ondisk_log_start":"117'6100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":10083,"log_dups_size":0,"ondisk_log_size":10083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16183,"num_write_kb":522944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":159},{"shards":"2,7","objects":-6914}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":579,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328657+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:25:36.328657+0000","last_peered":"2024-08-17T17:25:36.328657+0000","last_clean":"2024-08-17T17:25:36.328657+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:25:36.328657+0000","last_undegraded":"2024-08-17T17:25:36.328657+0000","last_fullsized":"2024-08-17T17:25:36.328657+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.330860+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:25:36.330860+0000","last_peered":"2024-08-17T17:25:36.330860+0000","last_clean":"2024-08-17T17:25:36.330860+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:25:36.330860+0000","last_undegraded":"2024-08-17T17:25:36.330860+0000","last_fullsized":"2024-08-17T17:25:36.330860+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.336988+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:25:36.336988+0000","last_peered":"2024-08-17T17:25:36.336988+0000","last_clean":"2024-08-17T17:25:36.336988+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:25:36.336988+0000","last_undegraded":"2024-08-17T17:25:36.336988+0000","last_fullsized":"2024-08-17T17:25:36.336988+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"161'16122","reported_seq":17283,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:38.448167+0000","last_change":"2024-08-17T17:22:35.146881+0000","last_active":"2024-08-17T17:25:38.448167+0000","last_peered":"2024-08-17T17:25:38.448167+0000","last_clean":"2024-08-17T17:22:02.019067+0000","last_became_active":"2024-08-17T17:22:21.260483+0000","last_became_peered":"2024-08-17T17:22:21.260483+0000","last_unstale":"2024-08-17T17:25:38.448167+0000","last_undegraded":"2024-08-17T17:22:35.146558+0000","last_fullsized":"2024-08-17T17:22:21.195836+0000","mapping_epoch":127,"log_start":"129'8771","ondisk_log_start":"129'8771","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16122,"num_write_kb":520064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":130}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":978,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:36.329326+0000","last_change":"2024-08-17T17:22:36.171266+0000","last_active":"2024-08-17T17:25:36.329326+0000","last_peered":"2024-08-17T17:25:36.329326+0000","last_clean":"2024-08-17T17:07:04.762546+0000","last_became_active":"2024-08-17T17:22:36.170391+0000","last_became_peered":"2024-08-17T17:22:36.170391+0000","last_unstale":"2024-08-17T17:25:36.329326+0000","last_undegraded":"2024-08-17T17:22:36.138225+0000","last_fullsized":"2024-08-17T17:22:36.137563+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277856+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:25:25.277856+0000","last_peered":"2024-08-17T17:25:25.277856+0000","last_clean":"2024-08-17T17:25:25.277856+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:25:25.277856+0000","last_undegraded":"2024-08-17T17:25:25.277856+0000","last_fullsized":"2024-08-17T17:25:25.277856+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.331632+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:25:36.331632+0000","last_peered":"2024-08-17T17:25:36.331632+0000","last_clean":"2024-08-17T17:25:36.331632+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:25:36.331632+0000","last_undegraded":"2024-08-17T17:25:36.331632+0000","last_fullsized":"2024-08-17T17:25:36.331632+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"161'16396","reported_seq":29214,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:38.460989+0000","last_change":"2024-08-17T17:23:24.130040+0000","last_active":"2024-08-17T17:25:38.460989+0000","last_peered":"2024-08-17T17:25:38.460989+0000","last_clean":"2024-08-17T17:25:38.460989+0000","last_became_active":"2024-08-17T17:23:24.107950+0000","last_became_peered":"2024-08-17T17:23:24.107950+0000","last_unstale":"2024-08-17T17:25:38.460989+0000","last_undegraded":"2024-08-17T17:25:38.460989+0000","last_fullsized":"2024-08-17T17:25:38.460989+0000","mapping_epoch":147,"log_start":"129'9033","ondisk_log_start":"129'9033","created":111,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.269300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9437184,"num_objects":144,"num_object_clones":0,"num_object_copies":432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16396,"num_write_kb":529280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6340,"num_bytes_recovered":415432704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":142,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326630+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:25:36.326630+0000","last_peered":"2024-08-17T17:25:36.326630+0000","last_clean":"2024-08-17T17:25:36.326630+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:25:36.326630+0000","last_undegraded":"2024-08-17T17:25:36.326630+0000","last_fullsized":"2024-08-17T17:25:36.326630+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":463,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.331963+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:25:36.331963+0000","last_peered":"2024-08-17T17:25:36.331963+0000","last_clean":"2024-08-17T17:25:36.331963+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:25:36.331963+0000","last_undegraded":"2024-08-17T17:25:36.331963+0000","last_fullsized":"2024-08-17T17:25:36.331963+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":169,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.329660+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:25:36.329660+0000","last_peered":"2024-08-17T17:25:36.329660+0000","last_clean":"2024-08-17T17:25:36.329660+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:25:36.329660+0000","last_undegraded":"2024-08-17T17:25:36.329660+0000","last_fullsized":"2024-08-17T17:25:36.329660+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"161'16222","reported_seq":16376,"reported_epoch":161,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:25:40.671192+0000","last_change":"2024-08-17T17:22:36.625305+0000","last_active":"2024-08-17T17:25:40.671192+0000","last_peered":"2024-08-17T17:25:40.671192+0000","last_clean":"2024-08-17T17:22:34.124009+0000","last_became_active":"2024-08-17T17:22:36.443474+0000","last_became_peered":"2024-08-17T17:22:36.443474+0000","last_unstale":"2024-08-17T17:25:40.671192+0000","last_undegraded":"2024-08-17T17:25:40.671192+0000","last_fullsized":"2024-08-17T17:25:40.671192+0000","mapping_epoch":135,"log_start":"120'8088","ondisk_log_start":"120'8088","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":8134,"log_dups_size":0,"ondisk_log_size":8134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:00.639822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3145728,"num_objects":48,"num_object_clones":0,"num_object_copies":144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":48,"num_objects_unfound":0,"num_objects_dirty":48,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16222,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":48}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":166,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.331814+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:25:36.331814+0000","last_peered":"2024-08-17T17:25:36.331814+0000","last_clean":"2024-08-17T17:25:36.331814+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:25:36.331814+0000","last_undegraded":"2024-08-17T17:25:36.331814+0000","last_fullsized":"2024-08-17T17:25:36.331814+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":687,"reported_epoch":160,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:25.281064+0000","last_change":"2024-08-17T17:22:22.204708+0000","last_active":"2024-08-17T17:25:25.281064+0000","last_peered":"2024-08-17T17:25:25.281064+0000","last_clean":"2024-08-17T17:22:13.162029+0000","last_became_active":"2024-08-17T17:22:22.204212+0000","last_became_peered":"2024-08-17T17:22:22.204212+0000","last_unstale":"2024-08-17T17:25:25.281064+0000","last_undegraded":"2024-08-17T17:22:22.199525+0000","last_fullsized":"2024-08-17T17:22:22.199012+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:43.853842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":165,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.361866+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:25:36.361866+0000","last_peered":"2024-08-17T17:25:36.361866+0000","last_clean":"2024-08-17T17:25:36.361866+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:25:36.361866+0000","last_undegraded":"2024-08-17T17:25:36.361866+0000","last_fullsized":"2024-08-17T17:25:36.361866+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"161'16308","reported_seq":16489,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.394609+0000","last_change":"2024-08-17T17:22:22.216745+0000","last_active":"2024-08-17T17:25:36.394609+0000","last_peered":"2024-08-17T17:25:36.394609+0000","last_clean":"2024-08-17T17:25:36.394609+0000","last_became_active":"2024-08-17T17:22:22.201301+0000","last_became_peered":"2024-08-17T17:22:22.201301+0000","last_unstale":"2024-08-17T17:25:36.394609+0000","last_undegraded":"2024-08-17T17:25:36.394609+0000","last_fullsized":"2024-08-17T17:25:36.394609+0000","mapping_epoch":128,"log_start":"139'9575","ondisk_log_start":"139'9575","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:50.530318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12320768,"num_objects":188,"num_object_clones":0,"num_object_copies":564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16308,"num_write_kb":527872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":176,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326649+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:25:36.326649+0000","last_peered":"2024-08-17T17:25:36.326649+0000","last_clean":"2024-08-17T17:25:36.326649+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:25:36.326649+0000","last_undegraded":"2024-08-17T17:25:36.326649+0000","last_fullsized":"2024-08-17T17:25:36.326649+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":658,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:36.329828+0000","last_change":"2024-08-17T17:22:22.218776+0000","last_active":"2024-08-17T17:25:36.329828+0000","last_peered":"2024-08-17T17:25:36.329828+0000","last_clean":"2024-08-17T17:22:13.175840+0000","last_became_active":"2024-08-17T17:22:22.218325+0000","last_became_peered":"2024-08-17T17:22:22.218325+0000","last_unstale":"2024-08-17T17:25:36.329828+0000","last_undegraded":"2024-08-17T17:22:22.198362+0000","last_fullsized":"2024-08-17T17:22:22.197790+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.327841+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:25:36.327841+0000","last_peered":"2024-08-17T17:25:36.327841+0000","last_clean":"2024-08-17T17:25:36.327841+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:25:36.327841+0000","last_undegraded":"2024-08-17T17:25:36.327841+0000","last_fullsized":"2024-08-17T17:25:36.327841+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"161'16501","reported_seq":17831,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:38.723025+0000","last_change":"2024-08-17T17:22:40.268294+0000","last_active":"2024-08-17T17:25:38.723025+0000","last_peered":"2024-08-17T17:25:38.723025+0000","last_clean":"2024-08-17T17:25:38.723025+0000","last_became_active":"2024-08-17T17:22:40.247002+0000","last_became_peered":"2024-08-17T17:22:40.247002+0000","last_unstale":"2024-08-17T17:25:38.723025+0000","last_undegraded":"2024-08-17T17:25:38.723025+0000","last_fullsized":"2024-08-17T17:25:38.723025+0000","mapping_epoch":139,"log_start":"129'8930","ondisk_log_start":"129'8930","created":111,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:21.777592+0000","last_clean_scrub_stamp":"2024-08-17T17:20:21.777592+0000","objects_scrubbed":0,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:11.964740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8323072,"num_objects":127,"num_object_clones":0,"num_object_copies":381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16501,"num_write_kb":532096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":600,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":194,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354236+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:25:36.354236+0000","last_peered":"2024-08-17T17:25:36.354236+0000","last_clean":"2024-08-17T17:25:36.354236+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:25:36.354236+0000","last_undegraded":"2024-08-17T17:25:36.354236+0000","last_fullsized":"2024-08-17T17:25:36.354236+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":404,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285602+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:25:25.285602+0000","last_peered":"2024-08-17T17:25:25.285602+0000","last_clean":"2024-08-17T17:25:25.285602+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:25:25.285602+0000","last_undegraded":"2024-08-17T17:25:25.285602+0000","last_fullsized":"2024-08-17T17:25:25.285602+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277689+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:25:25.277689+0000","last_peered":"2024-08-17T17:25:25.277689+0000","last_clean":"2024-08-17T17:25:25.277689+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:25:25.277689+0000","last_undegraded":"2024-08-17T17:25:25.277689+0000","last_fullsized":"2024-08-17T17:25:25.277689+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277922+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:25:25.277922+0000","last_peered":"2024-08-17T17:25:25.277922+0000","last_clean":"2024-08-17T17:25:25.277922+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:25:25.277922+0000","last_undegraded":"2024-08-17T17:25:25.277922+0000","last_fullsized":"2024-08-17T17:25:25.277922+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":55,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277958+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:25:25.277958+0000","last_peered":"2024-08-17T17:25:25.277958+0000","last_clean":"2024-08-17T17:25:25.277958+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:25:25.277958+0000","last_undegraded":"2024-08-17T17:25:25.277958+0000","last_fullsized":"2024-08-17T17:25:25.277958+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":159,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.329538+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:25:36.329538+0000","last_peered":"2024-08-17T17:25:36.329538+0000","last_clean":"2024-08-17T17:25:36.329538+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:25:36.329538+0000","last_undegraded":"2024-08-17T17:25:36.329538+0000","last_fullsized":"2024-08-17T17:25:36.329538+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":155,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201382+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:25:09.201382+0000","last_peered":"2024-08-17T17:25:09.201382+0000","last_clean":"2024-08-17T17:25:09.201382+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:25:09.201382+0000","last_undegraded":"2024-08-17T17:25:09.201382+0000","last_fullsized":"2024-08-17T17:25:09.201382+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":159,"state":"active+clean","last_fresh":"2024-08-17T17:25:09.201873+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:25:09.201873+0000","last_peered":"2024-08-17T17:25:09.201873+0000","last_clean":"2024-08-17T17:25:09.201873+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:25:09.201873+0000","last_undegraded":"2024-08-17T17:25:09.201873+0000","last_fullsized":"2024-08-17T17:25:09.201873+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328796+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:25:36.328796+0000","last_peered":"2024-08-17T17:25:36.328796+0000","last_clean":"2024-08-17T17:25:36.328796+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:25:36.328796+0000","last_undegraded":"2024-08-17T17:25:36.328796+0000","last_fullsized":"2024-08-17T17:25:36.328796+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354765+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:25:36.354765+0000","last_peered":"2024-08-17T17:25:36.354765+0000","last_clean":"2024-08-17T17:25:36.354765+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:25:36.354765+0000","last_undegraded":"2024-08-17T17:25:36.354765+0000","last_fullsized":"2024-08-17T17:25:36.354765+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":176,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328162+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:25:36.328162+0000","last_peered":"2024-08-17T17:25:36.328162+0000","last_clean":"2024-08-17T17:25:36.328162+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:25:36.328162+0000","last_undegraded":"2024-08-17T17:25:36.328162+0000","last_fullsized":"2024-08-17T17:25:36.328162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.285568+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:25:25.285568+0000","last_peered":"2024-08-17T17:25:25.285568+0000","last_clean":"2024-08-17T17:25:25.285568+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:25:25.285568+0000","last_undegraded":"2024-08-17T17:25:25.285568+0000","last_fullsized":"2024-08-17T17:25:25.285568+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":192,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.327945+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:25:36.327945+0000","last_peered":"2024-08-17T17:25:36.327945+0000","last_clean":"2024-08-17T17:25:36.327945+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:25:36.327945+0000","last_undegraded":"2024-08-17T17:25:36.327945+0000","last_fullsized":"2024-08-17T17:25:36.327945+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":169,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354402+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:25:36.354402+0000","last_peered":"2024-08-17T17:25:36.354402+0000","last_clean":"2024-08-17T17:25:36.354402+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:25:36.354402+0000","last_undegraded":"2024-08-17T17:25:36.354402+0000","last_fullsized":"2024-08-17T17:25:36.354402+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.329160+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:25:36.329160+0000","last_peered":"2024-08-17T17:25:36.329160+0000","last_clean":"2024-08-17T17:25:36.329160+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:25:36.329160+0000","last_undegraded":"2024-08-17T17:25:36.329160+0000","last_fullsized":"2024-08-17T17:25:36.329160+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326160+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:25:36.326160+0000","last_peered":"2024-08-17T17:25:36.326160+0000","last_clean":"2024-08-17T17:25:36.326160+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:25:36.326160+0000","last_undegraded":"2024-08-17T17:25:36.326160+0000","last_fullsized":"2024-08-17T17:25:36.326160+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277695+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:25:25.277695+0000","last_peered":"2024-08-17T17:25:25.277695+0000","last_clean":"2024-08-17T17:25:25.277695+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:25:25.277695+0000","last_undegraded":"2024-08-17T17:25:25.277695+0000","last_fullsized":"2024-08-17T17:25:25.277695+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":197,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.328803+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:25:36.328803+0000","last_peered":"2024-08-17T17:25:36.328803+0000","last_clean":"2024-08-17T17:25:36.328803+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:25:36.328803+0000","last_undegraded":"2024-08-17T17:25:36.328803+0000","last_fullsized":"2024-08-17T17:25:36.328803+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":175,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.361725+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:25:36.361725+0000","last_peered":"2024-08-17T17:25:36.361725+0000","last_clean":"2024-08-17T17:25:36.361725+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:25:36.361725+0000","last_undegraded":"2024-08-17T17:25:36.361725+0000","last_fullsized":"2024-08-17T17:25:36.361725+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277758+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:25:25.277758+0000","last_peered":"2024-08-17T17:25:25.277758+0000","last_clean":"2024-08-17T17:25:25.277758+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:25:25.277758+0000","last_undegraded":"2024-08-17T17:25:25.277758+0000","last_fullsized":"2024-08-17T17:25:25.277758+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":146,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277780+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:25:25.277780+0000","last_peered":"2024-08-17T17:25:25.277780+0000","last_clean":"2024-08-17T17:25:25.277780+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:25:25.277780+0000","last_undegraded":"2024-08-17T17:25:25.277780+0000","last_fullsized":"2024-08-17T17:25:25.277780+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277632+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:25:25.277632+0000","last_peered":"2024-08-17T17:25:25.277632+0000","last_clean":"2024-08-17T17:25:25.277632+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:25:25.277632+0000","last_undegraded":"2024-08-17T17:25:25.277632+0000","last_fullsized":"2024-08-17T17:25:25.277632+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":131,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.278060+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:25:25.278060+0000","last_peered":"2024-08-17T17:25:25.278060+0000","last_clean":"2024-08-17T17:25:25.278060+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:25:25.278060+0000","last_undegraded":"2024-08-17T17:25:25.278060+0000","last_fullsized":"2024-08-17T17:25:25.278060+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":194,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.329064+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:25:36.329064+0000","last_peered":"2024-08-17T17:25:36.329064+0000","last_clean":"2024-08-17T17:25:36.329064+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:25:36.329064+0000","last_undegraded":"2024-08-17T17:25:36.329064+0000","last_fullsized":"2024-08-17T17:25:36.329064+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":183,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.326532+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:25:36.326532+0000","last_peered":"2024-08-17T17:25:36.326532+0000","last_clean":"2024-08-17T17:25:36.326532+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:25:36.326532+0000","last_undegraded":"2024-08-17T17:25:36.326532+0000","last_fullsized":"2024-08-17T17:25:36.326532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":45,"reported_epoch":160,"state":"active+clean","last_fresh":"2024-08-17T17:25:25.277440+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:25:25.277440+0000","last_peered":"2024-08-17T17:25:25.277440+0000","last_clean":"2024-08-17T17:25:25.277440+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:25:25.277440+0000","last_undegraded":"2024-08-17T17:25:25.277440+0000","last_fullsized":"2024-08-17T17:25:25.277440+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":186,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-08-17T17:25:36.354719+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:25:36.354719+0000","last_peered":"2024-08-17T17:25:36.354719+0000","last_clean":"2024-08-17T17:25:36.354719+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:25:36.354719+0000","last_undegraded":"2024-08-17T17:25:36.354719+0000","last_fullsized":"2024-08-17T17:25:36.354719+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":162594840,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36456,"num_objects_misplaced":464,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":261402,"num_write_kb":8444225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19225,"num_bytes_recovered":1094320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2750951424,"data_stored":2750939208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131232,"ondisk_log_size":131232,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201545,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":228892,"kb_used_data":85396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":93516260,"statfs":{"total":95995035648,"available":95760650240,"internally_reserved":0,"allocated":87445504,"data_stored":86515761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":146930123},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463184,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122720,"kb_used_data":19416,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":93622432,"statfs":{"total":95995035648,"available":95869370368,"internally_reserved":0,"allocated":19881984,"data_stored":18956837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724820,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986455,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":354768,"kb_used_data":134984,"kb_used_omap":1,"kb_used_meta":219774,"kb_avail":93390384,"statfs":{"total":95995035648,"available":95631753216,"internally_reserved":0,"allocated":138223616,"data_stored":137305531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":225049030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435347,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155280,"kb_used_data":33288,"kb_used_omap":5,"kb_used_meta":121914,"kb_avail":93589872,"statfs":{"total":95995035648,"available":95836028928,"internally_reserved":0,"allocated":34086912,"data_stored":33172623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":124840009},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598873,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267672,"kb_used_data":65164,"kb_used_omap":7,"kb_used_meta":202424,"kb_avail":93477480,"statfs":{"total":95995035648,"available":95720939520,"internally_reserved":0,"allocated":66727936,"data_stored":65810031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":207282349},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,4,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762400,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":181192,"kb_used_data":46784,"kb_used_omap":7,"kb_used_meta":134392,"kb_avail":93563960,"statfs":{"total":95995035648,"available":95809495040,"internally_reserved":0,"allocated":47906816,"data_stored":47000305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":137617825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925926,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":331672,"kb_used_data":101200,"kb_used_omap":7,"kb_used_meta":230200,"kb_avail":93413480,"statfs":{"total":95995035648,"available":95655403520,"internally_reserved":0,"allocated":103628800,"data_stored":102707580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":235725404},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":77529088,"data_stored":77529088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":36503552,"data_stored":36503552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":58331136,"data_stored":58327064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":31916032,"data_stored":31916032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":126615552,"data_stored":126615552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2331316224,"data_stored":2331312152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12845056,"data_stored":12845056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":75894784,"data_stored":75890712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:25:42.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:25:42.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:42.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:42.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:42.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[104516]: pgmap v502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 160 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 36486/7530 objects degraded (484.542%); 464/7530 objects misplaced (6.162%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[104516]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[109246]: pgmap v502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 160 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 36486/7530 objects degraded (484.542%); 464/7530 objects misplaced (6.162%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[109246]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:42.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:42.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:42.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:42.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:42.787 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:42 smithi175 ceph-mon[101434]: pgmap v502: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 160 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 36486/7530 objects degraded (484.542%); 464/7530 objects misplaced (6.162%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:25:42.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:42 smithi175 ceph-mon[101434]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:42.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:42.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:42.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:42.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:42.817 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:25:42.818 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:25:42.818 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:25:42.818 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:25:42.818 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:25:42.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:42.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:42.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:42.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:43.085 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:43.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:43.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:43.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:43.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:43.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.435 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:43.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:43.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:25:43.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:43.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:43.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:43.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.742 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36486/7530 objects degraded (484.542%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:43.743 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36486/7530 objects degraded (484.542%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:43.769 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:43.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:43.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:43.894 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36486/7530 objects degraded (484.542%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:43.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:43.959 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:44.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:44.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:44.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:43] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:25:44.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:44.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:44.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:44.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:44.457 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:44.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:44.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:44.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:44 smithi064 ceph-mon[104516]: pgmap v503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 100 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 36517/4659 objects degraded (783.795%); 85/4659 objects misplaced (1.824%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:44.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:44 smithi064 ceph-mon[109246]: pgmap v503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 100 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 36517/4659 objects degraded (783.795%); 85/4659 objects misplaced (1.824%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:44.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:44.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:44.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:44.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:44 smithi175 ceph-mon[101434]: pgmap v503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 100 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 36517/4659 objects degraded (783.795%); 85/4659 objects misplaced (1.824%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:44.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.954 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:44.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.028 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-08-17T17:25:45.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-08-17T17:25:45.056 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:45.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:45.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:45.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:45.233 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:25:45.233 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:45.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:25:45.253 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:45.269 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:45.274 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:25:45.274 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:45.296 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:25:45.296 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:45.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:45.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:45.316 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:25:45.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:45.323 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:45.343 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:25:45.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:45.360 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:25:45.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:45.376 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:25:45.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:45.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:45.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.468 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:25:45.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/586533721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3147292534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:45.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/586533721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3147292534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:45.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:45.564 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:45.564 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:45.564 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/586533721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.564 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.564 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3147292534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:45.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:45.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:45.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:45.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:45.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:45.945 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:46.047 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:46.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:46.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:46.194 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:46.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:25:46.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:46.318 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:46.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:46.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:46.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:46.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:46.482 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:46.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:46.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[104516]: pgmap v504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 36517/606 objects degraded (6025.908%); 85/606 objects misplaced (14.026%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:46.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[104516]: osdmap e162: 8 total, 8 up, 7 in 2024-08-17T17:25:46.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[109246]: pgmap v504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 36517/606 objects degraded (6025.908%); 85/606 objects misplaced (14.026%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:46.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:46.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:46 smithi064 ceph-mon[109246]: osdmap e162: 8 total, 8 up, 7 in 2024-08-17T17:25:46.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:46 smithi175 ceph-mon[101434]: pgmap v504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 36517/606 objects degraded (6025.908%); 85/606 objects misplaced (14.026%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:25:46.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:46 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:46.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:46 smithi175 ceph-mon[101434]: osdmap e162: 8 total, 8 up, 7 in 2024-08-17T17:25:46.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:46.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:46.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:46.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:46.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:46.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:47.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:47.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:47.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:47.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:47.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:47.557 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:47.659 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:47.680 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:47.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:48.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:48.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:48.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:48.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.177 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:48.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:48.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:48.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:48.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:48.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:48.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:48.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:48.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:48.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:48.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:48 smithi175 ceph-mon[101434]: pgmap v506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36522/84 objects degraded (43478.571%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:48.672 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:48.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:48.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:48 smithi064 ceph-mon[104516]: pgmap v506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:48.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36522/84 objects degraded (43478.571%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:48.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:48 smithi064 ceph-mon[109246]: pgmap v506: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:48.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36522/84 objects degraded (43478.571%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:48.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:48.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:48.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:48.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:48.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:49.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:49.042 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:49.129 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:49.129 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 36522/84 objects degraded (43478.571%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":732,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":162,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915134,"num_in_osds":7,"osd_in_since":1723915354,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":78},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":28,"data_bytes":4790245,"bytes_used":1211363328,"bytes_avail":670753886208,"bytes_total":671965249536,"degraded_objects":36522,"degraded_total":84,"degraded_ratio":434.78571428571428,"recovering_objects_per_sec":70,"recovering_bytes_per_sec":1035008,"recovering_keys_per_sec":0,"num_objects_recovered":708,"num_bytes_recovered":10354688,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":454},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":23,"modified":"2024-08-17T17:19:54.695374+0000","services":{}},"progress_events":{}} 2024-08-17T17:25:49.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:25:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=sqlstore.transactions t=2024-08-17T17:25:48.85780711Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:25:49.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:49.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:49.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:49.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:49.518 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:49.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:49.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:49.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2555629603' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:49.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2555629603' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:49.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2555629603' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:25:49.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:49.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:49.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.914 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:49.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:50.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.038 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.140 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:50.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:50.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:50.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.285 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:50.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.386 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.407 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.479 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' removed 2024-08-17T17:25:50.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:50.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:50.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:50.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:25:50.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.710 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:25:50.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:25:50.729 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[104516]: pgmap v507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:50.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3487388697' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[109246]: pgmap v507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:50.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3487388697' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:50.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:50 smithi175 ceph-mon[101434]: pgmap v507: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 36522/84 objects degraded (43478.571%); 1011 KiB/s, 70 objects/s recovering 2024-08-17T17:25:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3487388697' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:25:50.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:50.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:50.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:50.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:51.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:51.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:51.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:51.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:51.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:51.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:51.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:51.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:51.480 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:25:51.481 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:25:51.481 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-08-17T17:25:51.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-08-17T17:25:51.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:51.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:51.574 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:51.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:51.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:51.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:51.718 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:51.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:25:51.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:51 smithi064 ceph-mon[104516]: osdmap e163: 8 total, 8 up, 7 in 2024-08-17T17:25:51.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:25:51.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:51 smithi064 ceph-mon[109246]: osdmap e163: 8 total, 8 up, 7 in 2024-08-17T17:25:51.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:51.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:25:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:51 smithi175 ceph-mon[101434]: osdmap e163: 8 total, 8 up, 7 in 2024-08-17T17:25:51.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:51.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:52.097 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:52.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:52.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:52.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:52.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:52.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:52.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:52.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:52.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:52 smithi175 ceph-mon[101434]: pgmap v509: 73 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 30/84 objects degraded (35.714%) 2024-08-17T17:25:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:52 smithi175 ceph-mon[101434]: osdmap e164: 8 total, 8 up, 7 in 2024-08-17T17:25:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:52 smithi064 ceph-mon[104516]: pgmap v509: 73 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 30/84 objects degraded (35.714%) 2024-08-17T17:25:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:52 smithi064 ceph-mon[104516]: osdmap e164: 8 total, 8 up, 7 in 2024-08-17T17:25:52.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:52 smithi064 ceph-mon[109246]: pgmap v509: 73 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 30/84 objects degraded (35.714%) 2024-08-17T17:25:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:52 smithi064 ceph-mon[109246]: osdmap e164: 8 total, 8 up, 7 in 2024-08-17T17:25:52.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:52.726 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:52.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:52.851 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:52.853 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:52.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:52.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:53.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.103 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:53.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:53.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:53.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:53.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:53.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:53.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:53.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:53.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:53 smithi064 ceph-mon[104516]: osdmap e165: 8 total, 8 up, 7 in 2024-08-17T17:25:53.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 30/84 objects degraded (35.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:53.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:53 smithi064 ceph-mon[109246]: osdmap e165: 8 total, 8 up, 7 in 2024-08-17T17:25:53.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 30/84 objects degraded (35.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:53.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:53 smithi175 ceph-mon[101434]: osdmap e165: 8 total, 8 up, 7 in 2024-08-17T17:25:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 30/84 objects degraded (35.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:53.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:53.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:54.031 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:25:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:25:53] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:25:54.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:54.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:54.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:54.321 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:54.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:54.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:54.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:54.478 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:54.559 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:25:54.559 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:54.562 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:25:54.562 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:54.578 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:25:54.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:54.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:54.594 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:25:54.594 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:54.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:54.611 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:25:54.612 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:54.628 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:25:54.629 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:54.645 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:25:54.646 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:54.662 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:25:54.662 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:54.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:54.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:54.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:54.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:54.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:54.857 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:54 smithi175 ceph-mon[101434]: pgmap v512: 73 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2024-08-17T17:25:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:54 smithi064 ceph-mon[104516]: pgmap v512: 73 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2024-08-17T17:25:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:54 smithi064 ceph-mon[109246]: pgmap v512: 73 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 18/84 objects degraded (21.429%) 2024-08-17T17:25:54.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:54.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:54.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:55.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:55.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:55.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:55.233 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:25:55.234 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:25:55.234 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:25:55.234 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:25:55.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:55.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:55.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:55.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:55.475 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:55.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:55.598 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:55.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:55.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:55.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:55.829 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1279454990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:55.830 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:55.830 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:55.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:55.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:55.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1279454990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1279454990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:25:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:55.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:56.095 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:56.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:56.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:56.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:56.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:56.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:56.607 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:56.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:56.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:56.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:56 smithi064 ceph-mon[104516]: pgmap v513: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:25:56.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:56 smithi064 ceph-mon[109246]: pgmap v513: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:25:56.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:56 smithi175 ceph-mon[101434]: pgmap v513: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 4.6 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:25:56.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:56.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:57.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:57.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:57.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:57.239 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:57.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:57.341 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:57.362 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:57.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:57.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:57.585 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:25:57.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:57.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:57.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:57.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:57.735 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:25:57.735 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:25:57.735 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:25:57.735 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:25:57.735 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:25:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:57.800 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/787624545' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.800 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:57 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:57.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:57.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:57.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/787624545' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:57 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/787624545' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:57 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:57.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:57.984 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:57.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:58.086 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:58.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:58.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.232 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:58.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:58.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:58.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:58.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:25:58.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:58.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:58.702 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:25:58.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[104516]: pgmap v514: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 640 KiB/s, 4 objects/s recovering 2024-08-17T17:25:58.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[104516]: osdmap e166: 8 total, 8 up, 7 in 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[109246]: pgmap v514: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 640 KiB/s, 4 objects/s recovering 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:58.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:25:58 smithi064 ceph-mon[109246]: osdmap e166: 8 total, 8 up, 7 in 2024-08-17T17:25:58.825 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:58 smithi175 ceph-mon[101434]: pgmap v514: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 640 KiB/s, 4 objects/s recovering 2024-08-17T17:25:58.825 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:58 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:25:58.825 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:58 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:58.825 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:58 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:58.825 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:25:58 smithi175 ceph-mon[101434]: osdmap e166: 8 total, 8 up, 7 in 2024-08-17T17:25:58.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:58.857 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:58.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:58.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:58.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:58.979 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:59.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:59.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:59.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:59.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:59.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:59.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:59.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:59.475 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:59.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:59.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:25:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:59.701 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:59.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:59.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:59.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:59.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:59.965 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:25:59.966 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:25:59.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:25:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:00.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:00.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:00.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:00.215 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:15441 5.2 5:4d04414a:::benchmark_data_smithi064_147470_object15440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-08-17T17:26:00.306 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 322.00818067799997, 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.101536945, 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 15441 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:26:00.307 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295288+0000", 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295286+0000", 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295359+0000", 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2199000000000004e-05 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295370+0000", 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1088e-05 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295377+0000", 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8519999999999998e-06 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.309 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295427+0000", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0229999999999998e-05 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295563+0000", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000136332 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.295712+0000", 2024-08-17T17:26:00.310 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014863800000000001 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.296447+0000", 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00073474800000000002 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.296811+0000", 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036494799999999998 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396739+0000", 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.099927929999999998 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396788+0000", 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8658000000000001e-05 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:38.396825+0000", 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6554000000000003e-05 2024-08-17T17:26:00.312 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:56690 5.2 5:4690ef43:::benchmark_data_smithi064_147470_object56689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 302.63672228899998, 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.10647835, 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.313 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 56690 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666746+0000", 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.314 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666744+0000", 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666809+0000", 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5030000000000006e-05 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.315 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666832+0000", 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2864999999999999e-05 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666840+0000", 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8669999999999994e-06 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.666889+0000", 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8934000000000003e-05 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.667015+0000", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012589900000000001 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.667157+0000", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014245699999999999 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.668637+0000", 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001479749 2024-08-17T17:26:00.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.669409+0000", 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00077196899999999995 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773138+0000", 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.103729397 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.318 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773187+0000", 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8717999999999997e-05 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:20:57.773224+0000", 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7245e-05 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:132257 5.2 5:4c68c9f0:::benchmark_data_smithi064_147470_object313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-08-17T17:26:00.319 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 234.882080266, 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.075403524999999999, 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 132257 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:26:00.320 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421388+0000", 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421387+0000", 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.321 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421392+0000", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.955e-06 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421403+0000", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0524e-05 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421409+0000", 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4779999999999996e-06 2024-08-17T17:26:00.322 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421478+0000", 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8938999999999998e-05 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421608+0000", 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012981900000000001 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.323 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.421724+0000", 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000116679 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.427301+0000", 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0055767539999999997 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.427368+0000", 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6888000000000001e-05 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.324 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496703+0000", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069335205999999996 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496749+0000", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5312999999999998e-05 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:05.496792+0000", 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2991999999999998e-05 2024-08-17T17:26:00.325 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140080 5.2 5:4c96634f:::benchmark_data_smithi064_147470_object8136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 221.394848642, 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071440704999999993, 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.326 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140080 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908620+0000", 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.327 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908619+0000", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908624+0000", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908635+0000", 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0871e-05 2024-08-17T17:26:00.328 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908640+0000", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4759999999999997e-06 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908678+0000", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7441999999999998e-05 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908770+0000", 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2101000000000005e-05 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.329 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.908860+0000", 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9926999999999995e-05 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.912365+0000", 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003505317 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.330 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.912701+0000", 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00033640999999999999 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.979959+0000", 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067257586999999994 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.980015+0000", 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6258000000000002e-05 2024-08-17T17:26:00.331 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.980060+0000", 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5207999999999998e-05 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140104 5.2 5:445a3862:::benchmark_data_smithi064_147470_object8160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 221.34087115599999, 2024-08-17T17:26:00.332 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067168221, 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140104 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.333 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962597+0000", 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962596+0000", 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.334 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:00.336 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962606+0000", 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0079e-05 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962624+0000", 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8082999999999999e-05 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.962632+0000", 2024-08-17T17:26:00.337 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3939999999999996e-06 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964115+0000", 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001482145 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964220+0000", 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105437 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.338 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.964332+0000", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011228 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.969804+0000", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.005471854 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:18.970808+0000", 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00100413 2024-08-17T17:26:00.339 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029688+0000", 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058880035999999997 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029725+0000", 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6606000000000002e-05 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.340 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.029765+0000", 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0444000000000001e-05 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140356 5.2 5:4d794467:::benchmark_data_smithi064_147470_object8412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 220.87578810599999, 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.073559667999999995, 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.341 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140356 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.342 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427680+0000", 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427679+0000", 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427687+0000", 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1550000000000007e-06 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.343 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427697+0000", 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8770000000000005e-06 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427704+0000", 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1350000000000003e-06 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.344 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427735+0000", 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0335999999999999e-05 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427823+0000", 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8755e-05 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.427913+0000", 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9384999999999999e-05 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.345 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.431013+0000", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030998760000000001 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.431694+0000", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068099499999999999 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501141+0000", 2024-08-17T17:26:00.346 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069447735999999996 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501193+0000", 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1779e-05 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.501240+0000", 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6662999999999997e-05 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.347 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140489 5.2 5:4dfa23d8:::benchmark_data_smithi064_147470_object8545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 220.618877239, 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.078668141999999996, 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140489 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684591+0000", 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.349 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684590+0000", 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684599+0000", 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9120000000000004e-06 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684611+0000", 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2181000000000001e-05 2024-08-17T17:26:00.350 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684619+0000", 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5719999999999997e-06 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684656+0000", 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7302999999999998e-05 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.351 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684739+0000", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2908e-05 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.684813+0000", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4202999999999994e-05 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.688043+0000", 2024-08-17T17:26:00.352 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032301790000000001 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.688172+0000", 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000128811 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763170+0000", 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.074998193000000005 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.353 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763216+0000", 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6176e-05 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:19.763259+0000", 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2843999999999998e-05 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.354 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140767 5.2 5:4475c63a:::benchmark_data_smithi064_147470_object8823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 220.15053646000001, 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.085237967999999997, 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140767 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.355 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152932+0000", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152931+0000", 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.356 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152937+0000", 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3400000000000003e-06 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152949+0000", 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1918e-05 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.357 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152955+0000", 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4420000000000003e-06 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.152998+0000", 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3167999999999997e-05 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.153095+0000", 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6910999999999994e-05 2024-08-17T17:26:00.358 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.153174+0000", 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8838999999999995e-05 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.156443+0000", 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032685489999999999 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.359 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.157288+0000", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00084548399999999997 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238061+0000", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.080772565000000004 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238120+0000", 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.914e-05 2024-08-17T17:26:00.360 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.238170+0000", 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9869e-05 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140772 5.2 5:4d35615e:::benchmark_data_smithi064_147470_object8828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.361 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 220.14187866, 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.085906341999999997, 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140772 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:26:00.362 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161590+0000", 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161579+0000", 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.99999 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.363 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161601+0000", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1097000000000001e-05 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161635+0000", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4703000000000003e-05 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161655+0000", 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9422000000000001e-05 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.364 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161780+0000", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000124845 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.161978+0000", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019837000000000001 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.162114+0000", 2024-08-17T17:26:00.365 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000136394 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.165681+0000", 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035668789999999998 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.165786+0000", 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105247 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.366 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247377+0000", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.081591061000000006 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247448+0000", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0835999999999997e-05 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247496+0000", 2024-08-17T17:26:00.367 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7707999999999998e-05 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140780 5.2 5:4421a3fd:::benchmark_data_smithi064_147470_object8836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 220.12315040600001, 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067303092999999994, 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.368 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140780 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180318+0000", 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.369 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180317+0000", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180323+0000", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0109999999999997e-06 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180337+0000", 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4303e-05 2024-08-17T17:26:00.370 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180346+0000", 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8599999999999999e-06 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180389+0000", 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3778e-05 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.371 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180521+0000", 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013125699999999999 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.180653+0000", 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013204699999999999 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.184422+0000", 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037690570000000001 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.372 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.184455+0000", 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3497000000000001e-05 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247540+0000", 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.063084514999999994 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.373 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247585+0000", 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5453e-05 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:20.247621+0000", 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5731000000000001e-05 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.374 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:163139 5.2 5:41aac34f:::benchmark_data_smithi064_147470_object31195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 180.61343927300001, 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.096410124, 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 163139 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.375 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690029+0000", 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.376 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690027+0000", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690037+0000", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0297e-05 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690057+0000", 2024-08-17T17:26:00.377 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0217999999999999e-05 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690067+0000", 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8689999999999999e-06 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690120+0000", 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3114e-05 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.378 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690229+0000", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010832699999999999 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.690325+0000", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6348000000000007e-05 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.693907+0000", 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035820880000000002 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.379 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.694092+0000", 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000184458 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.694644+0000", 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000552387 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.380 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786314+0000", 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.091670351999999997 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786364+0000", 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9913000000000001e-05 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:59.786439+0000", 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4970999999999998e-05 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.381 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:169887 5.2 5:4ed2308c:::benchmark_data_smithi064_147470_object37943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:11.157842+0000", 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 169.145625827, 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068662918000000003, 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.382 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 169887 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157842+0000", 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157842+0000", 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.383 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157841+0000", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157850+0000", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157865+0000", 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5440000000000001e-05 2024-08-17T17:26:00.384 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.157873+0000", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1939999999999998e-06 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.159472+0000", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0015984339999999999 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.159579+0000", 2024-08-17T17:26:00.385 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010705999999999999 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.159667+0000", 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8707000000000003e-05 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.163066+0000", 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033987449999999999 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.386 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.163902+0000", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00083567899999999998 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.215000+0000", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.051098001999999997 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.226311+0000", 2024-08-17T17:26:00.387 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011311351000000001 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.226375+0000", 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3855000000000004e-05 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:11.226505+0000", 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000130333 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.388 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:177137 5.2 5:492dcedf:::benchmark_data_smithi064_147470_object45193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:24.341818+0000", 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 155.961650416, 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068405545999999998, 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 177137 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.389 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341818+0000", 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341818+0000", 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.390 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341816+0000", 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341822+0000", 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2870000000000001e-06 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341837+0000", 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4854999999999999e-05 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.391 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341843+0000", 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.468e-06 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.341892+0000", 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9679999999999999e-05 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.392 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.342058+0000", 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016547799999999999 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.342240+0000", 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018210200000000001 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.346433+0000", 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041925970000000002 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.347114+0000", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068184799999999998 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.351667+0000", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004552838 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.409993+0000", 2024-08-17T17:26:00.394 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058325368000000002 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.410088+0000", 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5353999999999995e-05 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:24.410223+0000", 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000135473 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.395 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:177837 5.2 5:4f9ee710:::benchmark_data_smithi064_147470_object45893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:25.449285+0000", 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 154.854183629, 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.075625259, 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 177837 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.396 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449285+0000", 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449285+0000", 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.397 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449283+0000", 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449292+0000", 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8689999999999993e-06 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449309+0000", 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7632999999999999e-05 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.398 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449318+0000", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5890000000000003e-06 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449375+0000", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6997999999999999e-05 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449510+0000", 2024-08-17T17:26:00.399 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000135418 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.449642+0000", 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013185700000000001 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.454361+0000", 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0047189340000000001 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.400 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.454457+0000", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5565999999999999e-05 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.454534+0000", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7410999999999995e-05 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.524706+0000", 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.070171481999999993 2024-08-17T17:26:00.401 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.524768+0000", 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2669000000000006e-05 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:25.524910+0000", 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000141598 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.402 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:225524 5.2 5:48f6729a:::benchmark_data_smithi064_147470_object93580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:41.011053+0000", 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 79.292415794999997, 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.076567491000000001, 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 225524 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.403 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011053+0000", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011053+0000", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011051+0000", 2024-08-17T17:26:00.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011059+0000", 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5450000000000003e-06 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011082+0000", 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2314e-05 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.405 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011091+0000", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1099999999999992e-06 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011152+0000", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1273999999999996e-05 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011296+0000", 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014410700000000001 2024-08-17T17:26:00.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.011449+0000", 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000153218 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.017049+0000", 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0055993529999999996 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.407 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.017615+0000", 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00056628199999999998 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.025182+0000", 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0075672930000000001 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.087447+0000", 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.062265055999999999 2024-08-17T17:26:00.408 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.087509+0000", 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1653000000000001e-05 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:41.087620+0000", 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111146 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.409 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.34626.0:227310 5.f e157/135 5:fcc25aa5:::benchmark_data_smithi064_147470_object95366:head v 157'14347, mlcod=114'1126)", 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:43.843131+0000", 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 76.460337139000004, 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069086843999999994, 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843131+0000", 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.410 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843131+0000", 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843130+0000", 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.411 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843137+0000", 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843149+0000", 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2048000000000001e-05 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843158+0000", 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6070000000000008e-06 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.412 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843199+0000", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1449000000000002e-05 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.843223+0000", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3912000000000001e-05 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.912159+0000", 2024-08-17T17:26:00.413 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068936171000000004 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:43.912218+0000", 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8770999999999997e-05 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:228898 5.2 5:40881eb1:::benchmark_data_smithi064_147470_object96954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:46.283190+0000", 2024-08-17T17:26:00.414 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 74.020278207000004, 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068893159999999995, 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228898 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.415 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283190+0000", 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283190+0000", 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283189+0000", 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.416 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283200+0000", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1585e-05 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283219+0000", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8529000000000002e-05 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283226+0000", 2024-08-17T17:26:00.417 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3089999999999998e-06 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283286+0000", 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9565000000000003e-05 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283430+0000", 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014393399999999999 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.418 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.283568+0000", 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013803900000000001 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.287427+0000", 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038593999999999998 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.287496+0000", 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8943999999999996e-05 2024-08-17T17:26:00.419 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.298590+0000", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011094448 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.351893+0000", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.053302152999999998 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.351952+0000", 2024-08-17T17:26:00.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9293000000000003e-05 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.352083+0000", 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013147899999999999 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:228902 5.2 5:47d333d9:::benchmark_data_smithi064_147470_object96958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:46.290107+0000", 2024-08-17T17:26:00.421 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 74.013361313999994, 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.072026439999999997, 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228902 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.422 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290107+0000", 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290107+0000", 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290106+0000", 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.423 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290110+0000", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.873e-06 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290118+0000", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2139999999999996e-06 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.290124+0000", 2024-08-17T17:26:00.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9229999999999999e-06 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.294543+0000", 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00441917 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.425 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.426 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.426 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.294681+0000", 2024-08-17T17:26:00.426 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013719000000000001 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.294820+0000", 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000139161 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.303354+0000", 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0085342830000000001 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.427 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.303409+0000", 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4907999999999997e-05 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.307444+0000", 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040351500000000004 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.428 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.361986+0000", 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.054542286000000002 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.362027+0000", 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0572999999999999e-05 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.362133+0000", 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010649699999999999 2024-08-17T17:26:00.429 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:228907 5.2 5:43dff50f:::benchmark_data_smithi064_147470_object96963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:46.293494+0000", 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 74.009973986999995, 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068869024000000001, 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.430 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228907 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293494+0000", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293494+0000", 2024-08-17T17:26:00.431 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293493+0000", 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293501+0000", 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1280000000000008e-06 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.432 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293520+0000", 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8902e-05 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.293528+0000", 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0690000000000002e-06 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.298665+0000", 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051371760000000002 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.298822+0000", 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015661399999999999 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.298988+0000", 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000165635 2024-08-17T17:26:00.434 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.303393+0000", 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0044054020000000001 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.303417+0000", 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4360999999999999e-05 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.435 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.307581+0000", 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041634360000000004 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.362185+0000", 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.054604068999999998 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.362240+0000", 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5396999999999998e-05 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.436 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:46.362363+0000", 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012312799999999999 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:234782 5.2 5:450ebab2:::benchmark_data_smithi064_147470_object102838:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:55.515447+0000", 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 64.788021811999997, 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.072418660999999995, 2024-08-17T17:26:00.437 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 234782 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515447+0000", 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.438 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515447+0000", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515445+0000", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515452+0000", 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2200000000000003e-06 2024-08-17T17:26:00.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515465+0000", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3386e-05 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515472+0000", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0960000000000003e-06 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515525+0000", 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2352999999999998e-05 2024-08-17T17:26:00.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515643+0000", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011832699999999999 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.515764+0000", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120813 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.520110+0000", 2024-08-17T17:26:00.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043458280000000004 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.522380+0000", 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022704050000000001 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.525790+0000", 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0034101610000000001 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.587698+0000", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.061908154999999999 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.587767+0000", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8484000000000004e-05 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:55.587865+0000", 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8247999999999999e-05 2024-08-17T17:26:00.443 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.444 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.444 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.444 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:26:00.444 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:26:00.444 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:26:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:00.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:00.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:00.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:00 smithi175 ceph-mon[101434]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:00.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:00.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:00 smithi064 ceph-mon[104516]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:00 smithi064 ceph-mon[109246]: pgmap v516: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:00.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:00.792 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:00.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:00.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:00.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:01.063 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":516,"stamp":"2024-08-17T17:25:58.876306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":10165431,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":221,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1198068,"kb_used_data":32776,"kb_used_omap":33,"kb_used_meta":1165150,"kb_avail":655017996,"statfs":{"total":671965249536,"available":670738427904,"internally_reserved":0,"allocated":33562624,"data_stored":26941470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1193114187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.391212"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":187,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447345+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:25:58.447345+0000","last_peered":"2024-08-17T17:25:58.447345+0000","last_clean":"2024-08-17T17:25:58.447345+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:25:58.447345+0000","last_undegraded":"2024-08-17T17:25:58.447345+0000","last_fullsized":"2024-08-17T17:25:58.447345+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":159,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488731+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:25:51.488731+0000","last_peered":"2024-08-17T17:25:51.488731+0000","last_clean":"2024-08-17T17:25:51.488731+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:25:51.488731+0000","last_undegraded":"2024-08-17T17:25:51.488731+0000","last_fullsized":"2024-08-17T17:25:51.488731+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488915+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:25:51.488915+0000","last_peered":"2024-08-17T17:25:51.488915+0000","last_clean":"2024-08-17T17:25:51.488915+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:25:51.488915+0000","last_undegraded":"2024-08-17T17:25:51.488915+0000","last_fullsized":"2024-08-17T17:25:51.488915+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.572935+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:25:57.572935+0000","last_peered":"2024-08-17T17:25:57.572935+0000","last_clean":"2024-08-17T17:25:57.572935+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:25:57.572935+0000","last_undegraded":"2024-08-17T17:25:57.572935+0000","last_fullsized":"2024-08-17T17:25:57.572935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573051+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:25:57.573051+0000","last_peered":"2024-08-17T17:25:57.573051+0000","last_clean":"2024-08-17T17:25:57.573051+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:25:57.573051+0000","last_undegraded":"2024-08-17T17:25:57.573051+0000","last_fullsized":"2024-08-17T17:25:57.573051+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":130,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488777+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:25:51.488777+0000","last_peered":"2024-08-17T17:25:51.488777+0000","last_clean":"2024-08-17T17:25:51.488777+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:25:51.488777+0000","last_undegraded":"2024-08-17T17:25:51.488777+0000","last_fullsized":"2024-08-17T17:25:51.488777+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":186,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447585+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:25:58.447585+0000","last_peered":"2024-08-17T17:25:58.447585+0000","last_clean":"2024-08-17T17:25:58.447585+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:25:58.447585+0000","last_undegraded":"2024-08-17T17:25:58.447585+0000","last_fullsized":"2024-08-17T17:25:58.447585+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":154,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490329+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:25:51.490329+0000","last_peered":"2024-08-17T17:25:51.490329+0000","last_clean":"2024-08-17T17:25:51.490329+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:25:51.490329+0000","last_undegraded":"2024-08-17T17:25:51.490329+0000","last_fullsized":"2024-08-17T17:25:51.490329+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":154,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.489388+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:25:51.489388+0000","last_peered":"2024-08-17T17:25:51.489388+0000","last_clean":"2024-08-17T17:25:51.489388+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:25:51.489388+0000","last_undegraded":"2024-08-17T17:25:51.489388+0000","last_fullsized":"2024-08-17T17:25:51.489388+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488537+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:25:51.488537+0000","last_peered":"2024-08-17T17:25:51.488537+0000","last_clean":"2024-08-17T17:25:51.488537+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:25:51.488537+0000","last_undegraded":"2024-08-17T17:25:51.488537+0000","last_fullsized":"2024-08-17T17:25:51.488537+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":156,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.489269+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:25:51.489269+0000","last_peered":"2024-08-17T17:25:51.489269+0000","last_clean":"2024-08-17T17:25:51.489269+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:25:51.489269+0000","last_undegraded":"2024-08-17T17:25:51.489269+0000","last_fullsized":"2024-08-17T17:25:51.489269+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491586+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:25:51.491586+0000","last_peered":"2024-08-17T17:25:51.491586+0000","last_clean":"2024-08-17T17:25:51.491586+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:25:51.491586+0000","last_undegraded":"2024-08-17T17:25:51.491586+0000","last_fullsized":"2024-08-17T17:25:51.491586+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":127,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.489183+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:25:51.489183+0000","last_peered":"2024-08-17T17:25:51.489183+0000","last_clean":"2024-08-17T17:25:51.489183+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:25:51.489183+0000","last_undegraded":"2024-08-17T17:25:51.489183+0000","last_fullsized":"2024-08-17T17:25:51.489183+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490398+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:25:51.490398+0000","last_peered":"2024-08-17T17:25:51.490398+0000","last_clean":"2024-08-17T17:25:51.490398+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:25:51.490398+0000","last_undegraded":"2024-08-17T17:25:51.490398+0000","last_fullsized":"2024-08-17T17:25:51.490398+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447460+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:25:58.447460+0000","last_peered":"2024-08-17T17:25:58.447460+0000","last_clean":"2024-08-17T17:25:58.447460+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:25:58.447460+0000","last_undegraded":"2024-08-17T17:25:58.447460+0000","last_fullsized":"2024-08-17T17:25:58.447460+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":192,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573142+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:25:57.573142+0000","last_peered":"2024-08-17T17:25:57.573142+0000","last_clean":"2024-08-17T17:25:57.573142+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:25:57.573142+0000","last_undegraded":"2024-08-17T17:25:57.573142+0000","last_fullsized":"2024-08-17T17:25:57.573142+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":208,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447487+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:25:58.447487+0000","last_peered":"2024-08-17T17:25:58.447487+0000","last_clean":"2024-08-17T17:25:58.447487+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:25:58.447487+0000","last_undegraded":"2024-08-17T17:25:58.447487+0000","last_fullsized":"2024-08-17T17:25:58.447487+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491326+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:25:51.491326+0000","last_peered":"2024-08-17T17:25:51.491326+0000","last_clean":"2024-08-17T17:25:51.491326+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:25:51.491326+0000","last_undegraded":"2024-08-17T17:25:51.491326+0000","last_fullsized":"2024-08-17T17:25:51.491326+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490499+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:25:51.490499+0000","last_peered":"2024-08-17T17:25:51.490499+0000","last_clean":"2024-08-17T17:25:51.490499+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:25:51.490499+0000","last_undegraded":"2024-08-17T17:25:51.490499+0000","last_fullsized":"2024-08-17T17:25:51.490499+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491481+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:25:51.491481+0000","last_peered":"2024-08-17T17:25:51.491481+0000","last_clean":"2024-08-17T17:25:51.491481+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:25:51.491481+0000","last_undegraded":"2024-08-17T17:25:51.491481+0000","last_fullsized":"2024-08-17T17:25:51.491481+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490844+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:25:51.490844+0000","last_peered":"2024-08-17T17:25:51.490844+0000","last_clean":"2024-08-17T17:25:51.490844+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:25:51.490844+0000","last_undegraded":"2024-08-17T17:25:51.490844+0000","last_fullsized":"2024-08-17T17:25:51.490844+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491539+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:25:51.491539+0000","last_peered":"2024-08-17T17:25:51.491539+0000","last_clean":"2024-08-17T17:25:51.491539+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:25:51.491539+0000","last_undegraded":"2024-08-17T17:25:51.491539+0000","last_fullsized":"2024-08-17T17:25:51.491539+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":188,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447238+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:25:58.447238+0000","last_peered":"2024-08-17T17:25:58.447238+0000","last_clean":"2024-08-17T17:25:58.447238+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:25:58.447238+0000","last_undegraded":"2024-08-17T17:25:58.447238+0000","last_fullsized":"2024-08-17T17:25:58.447238+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":173,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.688604+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:25:52.688604+0000","last_peered":"2024-08-17T17:25:52.688604+0000","last_clean":"2024-08-17T17:25:52.688604+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:25:52.688604+0000","last_undegraded":"2024-08-17T17:25:52.688604+0000","last_fullsized":"2024-08-17T17:25:52.688604+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":459,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488645+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:25:51.488645+0000","last_peered":"2024-08-17T17:25:51.488645+0000","last_clean":"2024-08-17T17:25:51.488645+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:25:51.488645+0000","last_undegraded":"2024-08-17T17:25:51.488645+0000","last_fullsized":"2024-08-17T17:25:51.488645+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":701,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.688625+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:25:52.688625+0000","last_peered":"2024-08-17T17:25:52.688625+0000","last_clean":"2024-08-17T17:25:52.688625+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:25:52.688625+0000","last_undegraded":"2024-08-17T17:25:52.688625+0000","last_fullsized":"2024-08-17T17:25:52.688625+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447263+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:25:58.447263+0000","last_peered":"2024-08-17T17:25:58.447263+0000","last_clean":"2024-08-17T17:25:58.447263+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:25:58.447263+0000","last_undegraded":"2024-08-17T17:25:58.447263+0000","last_fullsized":"2024-08-17T17:25:58.447263+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":130,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490766+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:25:51.490766+0000","last_peered":"2024-08-17T17:25:51.490766+0000","last_clean":"2024-08-17T17:25:51.490766+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:25:51.490766+0000","last_undegraded":"2024-08-17T17:25:51.490766+0000","last_fullsized":"2024-08-17T17:25:51.490766+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":537,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.488535+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:25:51.488535+0000","last_peered":"2024-08-17T17:25:51.488535+0000","last_clean":"2024-08-17T17:25:51.488535+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:25:51.488535+0000","last_undegraded":"2024-08-17T17:25:51.488535+0000","last_fullsized":"2024-08-17T17:25:51.488535+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":193,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.572810+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:25:57.572810+0000","last_peered":"2024-08-17T17:25:57.572810+0000","last_clean":"2024-08-17T17:25:57.572810+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:25:57.572810+0000","last_undegraded":"2024-08-17T17:25:57.572810+0000","last_fullsized":"2024-08-17T17:25:57.572810+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":190,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573227+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:25:57.573227+0000","last_peered":"2024-08-17T17:25:57.573227+0000","last_clean":"2024-08-17T17:25:57.573227+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:25:57.573227+0000","last_undegraded":"2024-08-17T17:25:57.573227+0000","last_fullsized":"2024-08-17T17:25:57.573227+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":587,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.688381+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:25:52.688381+0000","last_peered":"2024-08-17T17:25:52.688381+0000","last_clean":"2024-08-17T17:25:52.688381+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:25:52.688381+0000","last_undegraded":"2024-08-17T17:25:52.688381+0000","last_fullsized":"2024-08-17T17:25:52.688381+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487479+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:25:52.487479+0000","last_peered":"2024-08-17T17:25:52.487479+0000","last_clean":"2024-08-17T17:25:52.487479+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:25:52.487479+0000","last_undegraded":"2024-08-17T17:25:52.487479+0000","last_fullsized":"2024-08-17T17:25:52.487479+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":208,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487927+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:25:52.487927+0000","last_peered":"2024-08-17T17:25:52.487927+0000","last_clean":"2024-08-17T17:25:52.487927+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:25:52.487927+0000","last_undegraded":"2024-08-17T17:25:52.487927+0000","last_fullsized":"2024-08-17T17:25:52.487927+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1015,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.490751+0000","last_change":"2024-08-17T17:25:52.490672+0000","last_active":"2024-08-17T17:25:52.490751+0000","last_peered":"2024-08-17T17:25:52.490751+0000","last_clean":"2024-08-17T17:25:52.490751+0000","last_became_active":"2024-08-17T17:25:52.490180+0000","last_became_peered":"2024-08-17T17:25:52.490180+0000","last_unstale":"2024-08-17T17:25:52.490751+0000","last_undegraded":"2024-08-17T17:25:52.490751+0000","last_fullsized":"2024-08-17T17:25:52.490751+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:46.014207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491173+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:25:51.491173+0000","last_peered":"2024-08-17T17:25:51.491173+0000","last_clean":"2024-08-17T17:25:51.491173+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:25:51.491173+0000","last_undegraded":"2024-08-17T17:25:51.491173+0000","last_fullsized":"2024-08-17T17:25:51.491173+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487653+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:25:52.487653+0000","last_peered":"2024-08-17T17:25:52.487653+0000","last_clean":"2024-08-17T17:25:52.487653+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:25:52.487653+0000","last_undegraded":"2024-08-17T17:25:52.487653+0000","last_fullsized":"2024-08-17T17:25:52.487653+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":152,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447645+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:25:58.447645+0000","last_peered":"2024-08-17T17:25:58.447645+0000","last_clean":"2024-08-17T17:25:58.447645+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:25:58.447645+0000","last_undegraded":"2024-08-17T17:25:58.447645+0000","last_fullsized":"2024-08-17T17:25:58.447645+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":471,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487811+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:25:52.487811+0000","last_peered":"2024-08-17T17:25:52.487811+0000","last_clean":"2024-08-17T17:25:52.487811+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:25:52.487811+0000","last_undegraded":"2024-08-17T17:25:52.487811+0000","last_fullsized":"2024-08-17T17:25:52.487811+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":177,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487673+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:25:52.487673+0000","last_peered":"2024-08-17T17:25:52.487673+0000","last_clean":"2024-08-17T17:25:52.487673+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:25:52.487673+0000","last_undegraded":"2024-08-17T17:25:52.487673+0000","last_fullsized":"2024-08-17T17:25:52.487673+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":174,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487688+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:25:52.487688+0000","last_peered":"2024-08-17T17:25:52.487688+0000","last_clean":"2024-08-17T17:25:52.487688+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:25:52.487688+0000","last_undegraded":"2024-08-17T17:25:52.487688+0000","last_fullsized":"2024-08-17T17:25:52.487688+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":727,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.491047+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:25:52.491047+0000","last_peered":"2024-08-17T17:25:52.491047+0000","last_clean":"2024-08-17T17:25:52.491047+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:25:52.491047+0000","last_undegraded":"2024-08-17T17:25:52.491047+0000","last_fullsized":"2024-08-17T17:25:52.491047+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":175,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573022+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:25:57.573022+0000","last_peered":"2024-08-17T17:25:57.573022+0000","last_clean":"2024-08-17T17:25:57.573022+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:25:57.573022+0000","last_undegraded":"2024-08-17T17:25:57.573022+0000","last_fullsized":"2024-08-17T17:25:57.573022+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":186,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447731+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:25:58.447731+0000","last_peered":"2024-08-17T17:25:58.447731+0000","last_clean":"2024-08-17T17:25:58.447731+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:25:58.447731+0000","last_undegraded":"2024-08-17T17:25:58.447731+0000","last_fullsized":"2024-08-17T17:25:58.447731+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":691,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.491802+0000","last_change":"2024-08-17T17:25:52.491743+0000","last_active":"2024-08-17T17:25:52.491802+0000","last_peered":"2024-08-17T17:25:52.491802+0000","last_clean":"2024-08-17T17:25:52.491802+0000","last_became_active":"2024-08-17T17:25:52.491147+0000","last_became_peered":"2024-08-17T17:25:52.491147+0000","last_unstale":"2024-08-17T17:25:52.491802+0000","last_undegraded":"2024-08-17T17:25:52.491802+0000","last_fullsized":"2024-08-17T17:25:52.491802+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:20.776398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":183,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447138+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:25:58.447138+0000","last_peered":"2024-08-17T17:25:58.447138+0000","last_clean":"2024-08-17T17:25:58.447138+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:25:58.447138+0000","last_undegraded":"2024-08-17T17:25:58.447138+0000","last_fullsized":"2024-08-17T17:25:58.447138+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":205,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573069+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:25:57.573069+0000","last_peered":"2024-08-17T17:25:57.573069+0000","last_clean":"2024-08-17T17:25:57.573069+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:25:57.573069+0000","last_undegraded":"2024-08-17T17:25:57.573069+0000","last_fullsized":"2024-08-17T17:25:57.573069+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":408,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490576+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:25:51.490576+0000","last_peered":"2024-08-17T17:25:51.490576+0000","last_clean":"2024-08-17T17:25:51.490576+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:25:51.490576+0000","last_undegraded":"2024-08-17T17:25:51.490576+0000","last_fullsized":"2024-08-17T17:25:51.490576+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491644+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:25:51.491644+0000","last_peered":"2024-08-17T17:25:51.491644+0000","last_clean":"2024-08-17T17:25:51.491644+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:25:51.491644+0000","last_undegraded":"2024-08-17T17:25:51.491644+0000","last_fullsized":"2024-08-17T17:25:51.491644+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491217+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:25:51.491217+0000","last_peered":"2024-08-17T17:25:51.491217+0000","last_clean":"2024-08-17T17:25:51.491217+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:25:51.491217+0000","last_undegraded":"2024-08-17T17:25:51.491217+0000","last_fullsized":"2024-08-17T17:25:51.491217+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":59,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491217+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:25:51.491217+0000","last_peered":"2024-08-17T17:25:51.491217+0000","last_clean":"2024-08-17T17:25:51.491217+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:25:51.491217+0000","last_undegraded":"2024-08-17T17:25:51.491217+0000","last_fullsized":"2024-08-17T17:25:51.491217+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487509+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:25:52.487509+0000","last_peered":"2024-08-17T17:25:52.487509+0000","last_clean":"2024-08-17T17:25:52.487509+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:25:52.487509+0000","last_undegraded":"2024-08-17T17:25:52.487509+0000","last_fullsized":"2024-08-17T17:25:52.487509+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":160,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.489014+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:25:51.489014+0000","last_peered":"2024-08-17T17:25:51.489014+0000","last_clean":"2024-08-17T17:25:51.489014+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:25:51.489014+0000","last_undegraded":"2024-08-17T17:25:51.489014+0000","last_fullsized":"2024-08-17T17:25:51.489014+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.489107+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:25:51.489107+0000","last_peered":"2024-08-17T17:25:51.489107+0000","last_clean":"2024-08-17T17:25:51.489107+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:25:51.489107+0000","last_undegraded":"2024-08-17T17:25:51.489107+0000","last_fullsized":"2024-08-17T17:25:51.489107+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":138,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447147+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:25:58.447147+0000","last_peered":"2024-08-17T17:25:58.447147+0000","last_clean":"2024-08-17T17:25:58.447147+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:25:58.447147+0000","last_undegraded":"2024-08-17T17:25:58.447147+0000","last_fullsized":"2024-08-17T17:25:58.447147+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":185,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573236+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:25:57.573236+0000","last_peered":"2024-08-17T17:25:57.573236+0000","last_clean":"2024-08-17T17:25:57.573236+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:25:57.573236+0000","last_undegraded":"2024-08-17T17:25:57.573236+0000","last_fullsized":"2024-08-17T17:25:57.573236+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":187,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447089+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:25:58.447089+0000","last_peered":"2024-08-17T17:25:58.447089+0000","last_clean":"2024-08-17T17:25:58.447089+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:25:58.447089+0000","last_undegraded":"2024-08-17T17:25:58.447089+0000","last_fullsized":"2024-08-17T17:25:58.447089+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490287+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:25:51.490287+0000","last_peered":"2024-08-17T17:25:51.490287+0000","last_clean":"2024-08-17T17:25:51.490287+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:25:51.490287+0000","last_undegraded":"2024-08-17T17:25:51.490287+0000","last_fullsized":"2024-08-17T17:25:51.490287+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":202,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447206+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:25:58.447206+0000","last_peered":"2024-08-17T17:25:58.447206+0000","last_clean":"2024-08-17T17:25:58.447206+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:25:58.447206+0000","last_undegraded":"2024-08-17T17:25:58.447206+0000","last_fullsized":"2024-08-17T17:25:58.447206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573147+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:25:57.573147+0000","last_peered":"2024-08-17T17:25:57.573147+0000","last_clean":"2024-08-17T17:25:57.573147+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:25:57.573147+0000","last_undegraded":"2024-08-17T17:25:57.573147+0000","last_fullsized":"2024-08-17T17:25:57.573147+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":181,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.487406+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:25:52.487406+0000","last_peered":"2024-08-17T17:25:52.487406+0000","last_clean":"2024-08-17T17:25:52.487406+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:25:52.487406+0000","last_undegraded":"2024-08-17T17:25:52.487406+0000","last_fullsized":"2024-08-17T17:25:52.487406+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447451+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:25:58.447451+0000","last_peered":"2024-08-17T17:25:58.447451+0000","last_clean":"2024-08-17T17:25:58.447451+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:25:58.447451+0000","last_undegraded":"2024-08-17T17:25:58.447451+0000","last_fullsized":"2024-08-17T17:25:58.447451+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491622+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:25:51.491622+0000","last_peered":"2024-08-17T17:25:51.491622+0000","last_clean":"2024-08-17T17:25:51.491622+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:25:51.491622+0000","last_undegraded":"2024-08-17T17:25:51.491622+0000","last_fullsized":"2024-08-17T17:25:51.491622+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":206,"reported_epoch":165,"state":"active+clean","last_fresh":"2024-08-17T17:25:52.688192+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:25:52.688192+0000","last_peered":"2024-08-17T17:25:52.688192+0000","last_clean":"2024-08-17T17:25:52.688192+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:25:52.688192+0000","last_undegraded":"2024-08-17T17:25:52.688192+0000","last_fullsized":"2024-08-17T17:25:52.688192+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":185,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573014+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:25:57.573014+0000","last_peered":"2024-08-17T17:25:57.573014+0000","last_clean":"2024-08-17T17:25:57.573014+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:25:57.573014+0000","last_undegraded":"2024-08-17T17:25:57.573014+0000","last_fullsized":"2024-08-17T17:25:57.573014+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491092+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:25:51.491092+0000","last_peered":"2024-08-17T17:25:51.491092+0000","last_clean":"2024-08-17T17:25:51.491092+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:25:51.491092+0000","last_undegraded":"2024-08-17T17:25:51.491092+0000","last_fullsized":"2024-08-17T17:25:51.491092+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":150,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491105+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:25:51.491105+0000","last_peered":"2024-08-17T17:25:51.491105+0000","last_clean":"2024-08-17T17:25:51.491105+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:25:51.491105+0000","last_undegraded":"2024-08-17T17:25:51.491105+0000","last_fullsized":"2024-08-17T17:25:51.491105+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491553+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:25:51.491553+0000","last_peered":"2024-08-17T17:25:51.491553+0000","last_clean":"2024-08-17T17:25:51.491553+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:25:51.491553+0000","last_undegraded":"2024-08-17T17:25:51.491553+0000","last_fullsized":"2024-08-17T17:25:51.491553+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":135,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.491253+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:25:51.491253+0000","last_peered":"2024-08-17T17:25:51.491253+0000","last_clean":"2024-08-17T17:25:51.491253+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:25:51.491253+0000","last_undegraded":"2024-08-17T17:25:51.491253+0000","last_fullsized":"2024-08-17T17:25:51.491253+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":204,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447454+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:25:58.447454+0000","last_peered":"2024-08-17T17:25:58.447454+0000","last_clean":"2024-08-17T17:25:58.447454+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:25:58.447454+0000","last_undegraded":"2024-08-17T17:25:58.447454+0000","last_fullsized":"2024-08-17T17:25:58.447454+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":193,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:58.447671+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:25:58.447671+0000","last_peered":"2024-08-17T17:25:58.447671+0000","last_clean":"2024-08-17T17:25:58.447671+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:25:58.447671+0000","last_undegraded":"2024-08-17T17:25:58.447671+0000","last_fullsized":"2024-08-17T17:25:58.447671+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":164,"state":"active+clean","last_fresh":"2024-08-17T17:25:51.490703+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:25:51.490703+0000","last_peered":"2024-08-17T17:25:51.490703+0000","last_clean":"2024-08-17T17:25:51.490703+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:25:51.490703+0000","last_undegraded":"2024-08-17T17:25:51.490703+0000","last_fullsized":"2024-08-17T17:25:51.490703+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":196,"reported_epoch":166,"state":"active+clean","last_fresh":"2024-08-17T17:25:57.573128+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:25:57.573128+0000","last_peered":"2024-08-17T17:25:57.573128+0000","last_clean":"2024-08-17T17:25:57.573128+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:25:57.573128+0000","last_undegraded":"2024-08-17T17:25:57.573128+0000","last_fullsized":"2024-08-17T17:25:57.573128+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201549,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":158376,"kb_used_data":6680,"kb_used_omap":1,"kb_used_meta":151678,"kb_avail":93586776,"statfs":{"total":95995035648,"available":95832858624,"internally_reserved":0,"allocated":6840320,"data_stored":5882198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":155318731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463187,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105892,"kb_used_data":2576,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":93639260,"statfs":{"total":95995035648,"available":95886602240,"internally_reserved":0,"allocated":2637824,"data_stored":1688084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724823,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986459,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":226584,"kb_used_data":10896,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93518568,"statfs":{"total":95995035648,"available":95763013632,"internally_reserved":0,"allocated":11157504,"data_stored":10206968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":220854726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435351,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123920,"kb_used_data":1984,"kb_used_omap":5,"kb_used_meta":121914,"kb_avail":93621232,"statfs":{"total":95995035648,"available":95868141568,"internally_reserved":0,"allocated":2031616,"data_stored":1092734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":124840009},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598877,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":200272,"kb_used_data":1984,"kb_used_omap":7,"kb_used_meta":198264,"kb_avail":93544880,"statfs":{"total":95995035648,"available":95789957120,"internally_reserved":0,"allocated":2031616,"data_stored":1093190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":203022509},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762403,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":137804,"kb_used_data":1976,"kb_used_omap":7,"kb_used_meta":135800,"kb_avail":93607348,"statfs":{"total":95995035648,"available":95853924352,"internally_reserved":0,"allocated":2023424,"data_stored":1092320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":139059617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925930,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245220,"kb_used_data":6680,"kb_used_omap":7,"kb_used_meta":238520,"kb_avail":93499932,"statfs":{"total":95995035648,"available":95743930368,"internally_reserved":0,"allocated":6840320,"data_stored":5885976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":244245084},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:26:01.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:01.090 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:01.150 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:01.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:01.208 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:26:01.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:01.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:01.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:01.410 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:01.438 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:01.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:01.519 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' created 2024-08-17T17:26:01.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:01.582 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:01.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:01 smithi064 ceph-mon[104516]: from='client.44942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:26:01.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:01 smithi064 ceph-mon[109246]: from='client.44942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:26:01.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:01.770 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:01 smithi175 ceph-mon[101434]: from='client.44942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:01.770 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:26:01.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:01.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:01.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:26:01.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:01.967 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:02.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:02.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:02.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.438 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:02.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:02.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:02.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:02.641 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:02 smithi175 ceph-mon[101434]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:02.641 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:26:02.641 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:02 smithi175 ceph-mon[101434]: osdmap e167: 8 total, 8 up, 7 in 2024-08-17T17:26:02.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:02.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:02.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:02.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:02.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[104516]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[104516]: osdmap e167: 8 total, 8 up, 7 in 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[109246]: pgmap v517: 73 pgs: 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 2 B/s, 2 objects/s recovering 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1556012096' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:26:02.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:02 smithi064 ceph-mon[109246]: osdmap e167: 8 total, 8 up, 7 in 2024-08-17T17:26:02.889 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:26:02.889 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:26:02.889 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:26:02.889 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:26:02.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:02.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:03.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:03.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:03.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-08-17T17:26:03.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:03.245 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:03.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:03.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:03.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:03.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:03.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:26:03.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:03.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:03.706 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:03.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:03.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:03.751 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:03.829 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:03 smithi175 ceph-mon[101434]: osdmap e168: 8 total, 8 up, 7 in 2024-08-17T17:26:03.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:03.864 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:03.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:03 smithi064 ceph-mon[104516]: osdmap e168: 8 total, 8 up, 7 in 2024-08-17T17:26:03.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:03 smithi064 ceph-mon[109246]: osdmap e168: 8 total, 8 up, 7 in 2024-08-17T17:26:03.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:03.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:04.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:04.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:04.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:04.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:04.116 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:03] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.51.0" 2024-08-17T17:26:04.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:04.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:04.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:04.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:04.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.488 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:04.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:04.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:04.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[104516]: pgmap v520: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[104516]: osdmap e169: 8 total, 8 up, 7 in 2024-08-17T17:26:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[109246]: pgmap v520: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:04 smithi064 ceph-mon[109246]: osdmap e169: 8 total, 8 up, 7 in 2024-08-17T17:26:04.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:04.734 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:04.802 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:04 smithi175 ceph-mon[101434]: pgmap v520: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:04.803 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:04 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:04.803 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:04 smithi175 ceph-mon[101434]: osdmap e169: 8 total, 8 up, 7 in 2024-08-17T17:26:04.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:04.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:04.887 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:26:04.887 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:04.898 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:26:04.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:04.917 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:26:04.917 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:04.935 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:26:04.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:04.952 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:26:04.952 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:04.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:04.968 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:26:04.969 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:04.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:04.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:26:04.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:05.001 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:26:05.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:05.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:05.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:05.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:05.208 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:05.230 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:05.315 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:05.316 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:05.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:05.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:05.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:05.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:05.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:05.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1615851681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:05.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1615851681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:05.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:05.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:05.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.868 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1615851681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:05.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:26:05.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:06.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:06.091 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:06.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:06.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:06.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:06.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:06.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:06.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:06.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:06.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:06.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:06.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:06.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:06.589 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:06.592 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:06.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:06.716 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:06.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:06.801 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:06 smithi064 ceph-mon[104516]: pgmap v522: 89 pgs: 6 unknown, 83 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:06.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:06 smithi064 ceph-mon[109246]: pgmap v522: 89 pgs: 6 unknown, 83 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:06.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:06.839 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:06 smithi175 ceph-mon[101434]: pgmap v522: 89 pgs: 6 unknown, 83 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:06.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:06.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:07.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:07.086 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:07.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:07.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:07.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:07.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:07.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:07.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:07.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:07.548 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:26:07.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:07.584 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4274848290' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:07 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.678 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:07.678 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:07.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:07.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:07.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:07.832 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:07.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:07.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4274848290' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:07 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4274848290' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:07 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:07.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:08.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:08.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.082 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:08.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:08.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:08.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:08.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:26:08.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:08.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:08.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:08.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:08.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:08.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.612 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:08.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:08 smithi175 ceph-mon[101434]: pgmap v523: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:08 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:08 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:08 smithi175 ceph-mon[101434]: osdmap e170: 8 total, 8 up, 7 in 2024-08-17T17:26:08.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[104516]: pgmap v523: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[104516]: osdmap e170: 8 total, 8 up, 7 in 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[109246]: pgmap v523: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:08 smithi064 ceph-mon[109246]: osdmap e170: 8 total, 8 up, 7 in 2024-08-17T17:26:08.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:08.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:08.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:08.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:08.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:08.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:09.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:09.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:09.211 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:09.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:09.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:09.357 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:09.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:09.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:09.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:09.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:09.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:09.731 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:09.833 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:09.857 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:09.959 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:09.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:10.072 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:10.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:10.107 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:10.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:10.231 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:10.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:10.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:10.457 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:10.465 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:26:10.466 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:26:10.480 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:10.582 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:10.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:10.706 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:10.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:10.831 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:10.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[104516]: pgmap v525: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:10.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:10.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:10.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:10.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[109246]: pgmap v525: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:10.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:10.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:10.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:10.853 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:10 smithi175 ceph-mon[101434]: pgmap v525: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:10.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:10.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:11.038 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-08-17T17:26:11.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:26:11.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:11.480 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:11.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[104516]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[104516]: osdmap e171: 8 total, 8 up, 7 in 2024-08-17T17:26:11.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[109246]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:11.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:11.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:11 smithi064 ceph-mon[109246]: osdmap e171: 8 total, 8 up, 7 in 2024-08-17T17:26:11.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.853 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:11 smithi175 ceph-mon[101434]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:11 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4147591978' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:11 smithi175 ceph-mon[101434]: osdmap e171: 8 total, 8 up, 7 in 2024-08-17T17:26:11.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:12.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:12.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:12.125 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":525,"stamp":"2024-08-17T17:26:08.878766+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":10165431,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":257,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1193572,"kb_used_data":28288,"kb_used_omap":33,"kb_used_meta":1165150,"kb_avail":655022492,"statfs":{"total":671965249536,"available":670743031808,"internally_reserved":0,"allocated":28966912,"data_stored":22241952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1193114187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.357595"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937261+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:26:07.937261+0000","last_peered":"2024-08-17T17:26:07.937261+0000","last_clean":"2024-08-17T17:26:07.937261+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:26:07.937261+0000","last_undegraded":"2024-08-17T17:26:07.937261+0000","last_fullsized":"2024-08-17T17:26:07.937261+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":163,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526329+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:26:02.526329+0000","last_peered":"2024-08-17T17:26:02.526329+0000","last_clean":"2024-08-17T17:26:02.526329+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:26:02.526329+0000","last_undegraded":"2024-08-17T17:26:02.526329+0000","last_fullsized":"2024-08-17T17:26:02.526329+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":106,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526691+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:26:02.526691+0000","last_peered":"2024-08-17T17:26:02.526691+0000","last_clean":"2024-08-17T17:26:02.526691+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:26:02.526691+0000","last_undegraded":"2024-08-17T17:26:02.526691+0000","last_fullsized":"2024-08-17T17:26:02.526691+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":186,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.974961+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:26:07.974961+0000","last_peered":"2024-08-17T17:26:07.974961+0000","last_clean":"2024-08-17T17:26:07.974961+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:26:07.974961+0000","last_undegraded":"2024-08-17T17:26:07.974961+0000","last_fullsized":"2024-08-17T17:26:07.974961+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":186,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975481+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:26:07.975481+0000","last_peered":"2024-08-17T17:26:07.975481+0000","last_clean":"2024-08-17T17:26:07.975481+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:26:07.975481+0000","last_undegraded":"2024-08-17T17:26:07.975481+0000","last_fullsized":"2024-08-17T17:26:07.975481+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":134,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526000+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:26:02.526000+0000","last_peered":"2024-08-17T17:26:02.526000+0000","last_clean":"2024-08-17T17:26:02.526000+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:26:02.526000+0000","last_undegraded":"2024-08-17T17:26:02.526000+0000","last_fullsized":"2024-08-17T17:26:02.526000+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937648+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:26:07.937648+0000","last_peered":"2024-08-17T17:26:07.937648+0000","last_clean":"2024-08-17T17:26:07.937648+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:26:07.937648+0000","last_undegraded":"2024-08-17T17:26:07.937648+0000","last_fullsized":"2024-08-17T17:26:07.937648+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":158,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526298+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:26:02.526298+0000","last_peered":"2024-08-17T17:26:02.526298+0000","last_clean":"2024-08-17T17:26:02.526298+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:26:02.526298+0000","last_undegraded":"2024-08-17T17:26:02.526298+0000","last_fullsized":"2024-08-17T17:26:02.526298+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:03.888529+0000","last_change":"2024-08-17T17:26:02.534643+0000","last_active":"2024-08-17T17:26:03.888529+0000","last_peered":"2024-08-17T17:26:03.888529+0000","last_clean":"2024-08-17T17:26:03.888529+0000","last_became_active":"2024-08-17T17:26:02.534394+0000","last_became_peered":"2024-08-17T17:26:02.534394+0000","last_unstale":"2024-08-17T17:26:03.888529+0000","last_undegraded":"2024-08-17T17:26:03.888529+0000","last_fullsized":"2024-08-17T17:26:03.888529+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:29.181237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":159,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526783+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:26:02.526783+0000","last_peered":"2024-08-17T17:26:02.526783+0000","last_clean":"2024-08-17T17:26:02.526783+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:26:02.526783+0000","last_undegraded":"2024-08-17T17:26:02.526783+0000","last_fullsized":"2024-08-17T17:26:02.526783+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.525928+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:26:02.525928+0000","last_peered":"2024-08-17T17:26:02.525928+0000","last_clean":"2024-08-17T17:26:02.525928+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:26:02.525928+0000","last_undegraded":"2024-08-17T17:26:02.525928+0000","last_fullsized":"2024-08-17T17:26:02.525928+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937130+0000","last_change":"2024-08-17T17:26:02.539050+0000","last_active":"2024-08-17T17:26:07.937130+0000","last_peered":"2024-08-17T17:26:07.937130+0000","last_clean":"2024-08-17T17:26:07.937130+0000","last_became_active":"2024-08-17T17:26:02.538574+0000","last_became_peered":"2024-08-17T17:26:02.538574+0000","last_unstale":"2024-08-17T17:26:07.937130+0000","last_undegraded":"2024-08-17T17:26:07.937130+0000","last_fullsized":"2024-08-17T17:26:07.937130+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:42.972791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":160,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526431+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:26:02.526431+0000","last_peered":"2024-08-17T17:26:02.526431+0000","last_clean":"2024-08-17T17:26:02.526431+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:26:02.526431+0000","last_undegraded":"2024-08-17T17:26:02.526431+0000","last_fullsized":"2024-08-17T17:26:02.526431+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522034+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:26:02.522034+0000","last_peered":"2024-08-17T17:26:02.522034+0000","last_clean":"2024-08-17T17:26:02.522034+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:26:02.522034+0000","last_undegraded":"2024-08-17T17:26:02.522034+0000","last_fullsized":"2024-08-17T17:26:02.522034+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:04.248317+0000","last_change":"2024-08-17T17:26:02.538436+0000","last_active":"2024-08-17T17:26:04.248317+0000","last_peered":"2024-08-17T17:26:04.248317+0000","last_clean":"2024-08-17T17:26:04.248317+0000","last_became_active":"2024-08-17T17:26:02.538195+0000","last_became_peered":"2024-08-17T17:26:02.538195+0000","last_unstale":"2024-08-17T17:26:04.248317+0000","last_undegraded":"2024-08-17T17:26:04.248317+0000","last_fullsized":"2024-08-17T17:26:04.248317+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:29.623332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":131,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526933+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:26:02.526933+0000","last_peered":"2024-08-17T17:26:02.526933+0000","last_clean":"2024-08-17T17:26:02.526933+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:26:02.526933+0000","last_undegraded":"2024-08-17T17:26:02.526933+0000","last_fullsized":"2024-08-17T17:26:02.526933+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526460+0000","last_change":"2024-08-17T17:17:50.474968+0000","last_active":"2024-08-17T17:26:02.526460+0000","last_peered":"2024-08-17T17:26:02.526460+0000","last_clean":"2024-08-17T17:26:02.526460+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:26:02.526460+0000","last_undegraded":"2024-08-17T17:26:02.526460+0000","last_fullsized":"2024-08-17T17:26:02.526460+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:44.587495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.974840+0000","last_change":"2024-08-17T17:26:02.538250+0000","last_active":"2024-08-17T17:26:07.974840+0000","last_peered":"2024-08-17T17:26:07.974840+0000","last_clean":"2024-08-17T17:26:07.974840+0000","last_became_active":"2024-08-17T17:26:02.537877+0000","last_became_peered":"2024-08-17T17:26:02.537877+0000","last_unstale":"2024-08-17T17:26:07.974840+0000","last_undegraded":"2024-08-17T17:26:07.974840+0000","last_fullsized":"2024-08-17T17:26:07.974840+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:31.846402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937377+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:26:07.937377+0000","last_peered":"2024-08-17T17:26:07.937377+0000","last_clean":"2024-08-17T17:26:07.937377+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:26:07.937377+0000","last_undegraded":"2024-08-17T17:26:07.937377+0000","last_fullsized":"2024-08-17T17:26:07.937377+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":200,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975399+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:26:07.975399+0000","last_peered":"2024-08-17T17:26:07.975399+0000","last_clean":"2024-08-17T17:26:07.975399+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:26:07.975399+0000","last_undegraded":"2024-08-17T17:26:07.975399+0000","last_fullsized":"2024-08-17T17:26:07.975399+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:04.249006+0000","last_change":"2024-08-17T17:26:02.533130+0000","last_active":"2024-08-17T17:26:04.249006+0000","last_peered":"2024-08-17T17:26:04.249006+0000","last_clean":"2024-08-17T17:26:04.249006+0000","last_became_active":"2024-08-17T17:26:02.532783+0000","last_became_peered":"2024-08-17T17:26:02.532783+0000","last_unstale":"2024-08-17T17:26:04.249006+0000","last_undegraded":"2024-08-17T17:26:04.249006+0000","last_fullsized":"2024-08-17T17:26:04.249006+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:32.674058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":216,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937236+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:26:07.937236+0000","last_peered":"2024-08-17T17:26:07.937236+0000","last_clean":"2024-08-17T17:26:07.937236+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:26:07.937236+0000","last_undegraded":"2024-08-17T17:26:07.937236+0000","last_fullsized":"2024-08-17T17:26:07.937236+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522312+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:26:02.522312+0000","last_peered":"2024-08-17T17:26:02.522312+0000","last_clean":"2024-08-17T17:26:02.522312+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:26:02.522312+0000","last_undegraded":"2024-08-17T17:26:02.522312+0000","last_fullsized":"2024-08-17T17:26:02.522312+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975102+0000","last_change":"2024-08-17T17:26:02.537749+0000","last_active":"2024-08-17T17:26:07.975102+0000","last_peered":"2024-08-17T17:26:07.975102+0000","last_clean":"2024-08-17T17:26:07.975102+0000","last_became_active":"2024-08-17T17:26:02.537124+0000","last_became_peered":"2024-08-17T17:26:02.537124+0000","last_unstale":"2024-08-17T17:26:07.975102+0000","last_undegraded":"2024-08-17T17:26:07.975102+0000","last_fullsized":"2024-08-17T17:26:07.975102+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:03.860631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":150,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526532+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:26:02.526532+0000","last_peered":"2024-08-17T17:26:02.526532+0000","last_clean":"2024-08-17T17:26:02.526532+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:26:02.526532+0000","last_undegraded":"2024-08-17T17:26:02.526532+0000","last_fullsized":"2024-08-17T17:26:02.526532+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522625+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:26:02.522625+0000","last_peered":"2024-08-17T17:26:02.522625+0000","last_clean":"2024-08-17T17:26:02.522625+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:26:02.522625+0000","last_undegraded":"2024-08-17T17:26:02.522625+0000","last_fullsized":"2024-08-17T17:26:02.522625+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937177+0000","last_change":"2024-08-17T17:26:02.538371+0000","last_active":"2024-08-17T17:26:07.937177+0000","last_peered":"2024-08-17T17:26:07.937177+0000","last_clean":"2024-08-17T17:26:07.937177+0000","last_became_active":"2024-08-17T17:26:02.537919+0000","last_became_peered":"2024-08-17T17:26:02.537919+0000","last_unstale":"2024-08-17T17:26:07.937177+0000","last_undegraded":"2024-08-17T17:26:07.937177+0000","last_fullsized":"2024-08-17T17:26:07.937177+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:30.702818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.521973+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:26:02.521973+0000","last_peered":"2024-08-17T17:26:02.521973+0000","last_clean":"2024-08-17T17:26:02.521973+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:26:02.521973+0000","last_undegraded":"2024-08-17T17:26:02.521973+0000","last_fullsized":"2024-08-17T17:26:02.521973+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522677+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:26:02.522677+0000","last_peered":"2024-08-17T17:26:02.522677+0000","last_clean":"2024-08-17T17:26:02.522677+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:26:02.522677+0000","last_undegraded":"2024-08-17T17:26:02.522677+0000","last_fullsized":"2024-08-17T17:26:02.522677+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":11,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.537952+0000","last_change":"2024-08-17T17:26:02.536813+0000","last_active":"2024-08-17T17:26:02.537952+0000","last_peered":"2024-08-17T17:26:02.537952+0000","last_clean":"2024-08-17T17:26:02.537952+0000","last_became_active":"2024-08-17T17:26:02.535602+0000","last_became_peered":"2024-08-17T17:26:02.535602+0000","last_unstale":"2024-08-17T17:26:02.537952+0000","last_undegraded":"2024-08-17T17:26:02.537952+0000","last_fullsized":"2024-08-17T17:26:02.537952+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:16:05.415637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":196,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937015+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:26:07.937015+0000","last_peered":"2024-08-17T17:26:07.937015+0000","last_clean":"2024-08-17T17:26:07.937015+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:26:07.937015+0000","last_undegraded":"2024-08-17T17:26:07.937015+0000","last_fullsized":"2024-08-17T17:26:07.937015+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975920+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:26:07.975920+0000","last_peered":"2024-08-17T17:26:07.975920+0000","last_clean":"2024-08-17T17:26:07.975920+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:26:07.975920+0000","last_undegraded":"2024-08-17T17:26:07.975920+0000","last_fullsized":"2024-08-17T17:26:07.975920+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937572+0000","last_change":"2024-08-17T17:26:02.536026+0000","last_active":"2024-08-17T17:26:07.937572+0000","last_peered":"2024-08-17T17:26:07.937572+0000","last_clean":"2024-08-17T17:26:07.937572+0000","last_became_active":"2024-08-17T17:26:02.535621+0000","last_became_peered":"2024-08-17T17:26:02.535621+0000","last_unstale":"2024-08-17T17:26:07.937572+0000","last_undegraded":"2024-08-17T17:26:07.937572+0000","last_fullsized":"2024-08-17T17:26:07.937572+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:48.950778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":464,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526032+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:26:02.526032+0000","last_peered":"2024-08-17T17:26:02.526032+0000","last_clean":"2024-08-17T17:26:02.526032+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:26:02.526032+0000","last_undegraded":"2024-08-17T17:26:02.526032+0000","last_fullsized":"2024-08-17T17:26:02.526032+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":711,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975817+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:26:07.975817+0000","last_peered":"2024-08-17T17:26:07.975817+0000","last_clean":"2024-08-17T17:26:07.975817+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:26:07.975817+0000","last_undegraded":"2024-08-17T17:26:07.975817+0000","last_fullsized":"2024-08-17T17:26:07.975817+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":153,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.936865+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:26:07.936865+0000","last_peered":"2024-08-17T17:26:07.936865+0000","last_clean":"2024-08-17T17:26:07.936865+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:26:07.936865+0000","last_undegraded":"2024-08-17T17:26:07.936865+0000","last_fullsized":"2024-08-17T17:26:07.936865+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526853+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:26:02.526853+0000","last_peered":"2024-08-17T17:26:02.526853+0000","last_clean":"2024-08-17T17:26:02.526853+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:26:02.526853+0000","last_undegraded":"2024-08-17T17:26:02.526853+0000","last_fullsized":"2024-08-17T17:26:02.526853+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":11,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.540495+0000","last_change":"2024-08-17T17:26:02.539873+0000","last_active":"2024-08-17T17:26:02.540495+0000","last_peered":"2024-08-17T17:26:02.540495+0000","last_clean":"2024-08-17T17:26:02.540495+0000","last_became_active":"2024-08-17T17:26:02.539492+0000","last_became_peered":"2024-08-17T17:26:02.539492+0000","last_unstale":"2024-08-17T17:26:02.540495+0000","last_undegraded":"2024-08-17T17:26:02.540495+0000","last_fullsized":"2024-08-17T17:26:02.540495+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:19.047224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":541,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.525929+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:26:02.525929+0000","last_peered":"2024-08-17T17:26:02.525929+0000","last_clean":"2024-08-17T17:26:02.525929+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:26:02.525929+0000","last_undegraded":"2024-08-17T17:26:02.525929+0000","last_fullsized":"2024-08-17T17:26:02.525929+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.974774+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:26:07.974774+0000","last_peered":"2024-08-17T17:26:07.974774+0000","last_clean":"2024-08-17T17:26:07.974774+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:26:07.974774+0000","last_undegraded":"2024-08-17T17:26:07.974774+0000","last_fullsized":"2024-08-17T17:26:07.974774+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":198,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975571+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:26:07.975571+0000","last_peered":"2024-08-17T17:26:07.975571+0000","last_clean":"2024-08-17T17:26:07.975571+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:26:07.975571+0000","last_undegraded":"2024-08-17T17:26:07.975571+0000","last_fullsized":"2024-08-17T17:26:07.975571+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:04.248542+0000","last_change":"2024-08-17T17:26:02.532201+0000","last_active":"2024-08-17T17:26:04.248542+0000","last_peered":"2024-08-17T17:26:04.248542+0000","last_clean":"2024-08-17T17:26:04.248542+0000","last_became_active":"2024-08-17T17:26:02.531942+0000","last_became_peered":"2024-08-17T17:26:02.531942+0000","last_unstale":"2024-08-17T17:26:04.248542+0000","last_undegraded":"2024-08-17T17:26:04.248542+0000","last_fullsized":"2024-08-17T17:26:04.248542+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:51.278560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":597,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975102+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:26:07.975102+0000","last_peered":"2024-08-17T17:26:07.975102+0000","last_clean":"2024-08-17T17:26:07.975102+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:26:07.975102+0000","last_undegraded":"2024-08-17T17:26:07.975102+0000","last_fullsized":"2024-08-17T17:26:07.975102+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":174,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526983+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:26:02.526983+0000","last_peered":"2024-08-17T17:26:02.526983+0000","last_clean":"2024-08-17T17:26:02.526983+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:26:02.526983+0000","last_undegraded":"2024-08-17T17:26:02.526983+0000","last_fullsized":"2024-08-17T17:26:02.526983+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":214,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.528727+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:26:02.528727+0000","last_peered":"2024-08-17T17:26:02.528727+0000","last_clean":"2024-08-17T17:26:02.528727+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:26:02.528727+0000","last_undegraded":"2024-08-17T17:26:02.528727+0000","last_fullsized":"2024-08-17T17:26:02.528727+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937488+0000","last_change":"2024-08-17T17:26:02.539405+0000","last_active":"2024-08-17T17:26:07.937488+0000","last_peered":"2024-08-17T17:26:07.937488+0000","last_clean":"2024-08-17T17:26:07.937488+0000","last_became_active":"2024-08-17T17:26:02.539076+0000","last_became_peered":"2024-08-17T17:26:02.539076+0000","last_unstale":"2024-08-17T17:26:07.937488+0000","last_undegraded":"2024-08-17T17:26:07.937488+0000","last_fullsized":"2024-08-17T17:26:07.937488+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:01:22.303577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1021,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526597+0000","last_change":"2024-08-17T17:25:52.490672+0000","last_active":"2024-08-17T17:26:02.526597+0000","last_peered":"2024-08-17T17:26:02.526597+0000","last_clean":"2024-08-17T17:26:02.526597+0000","last_became_active":"2024-08-17T17:25:52.490180+0000","last_became_peered":"2024-08-17T17:25:52.490180+0000","last_unstale":"2024-08-17T17:26:02.526597+0000","last_undegraded":"2024-08-17T17:26:02.526597+0000","last_fullsized":"2024-08-17T17:26:02.526597+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:46.014207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522337+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:26:02.522337+0000","last_peered":"2024-08-17T17:26:02.522337+0000","last_clean":"2024-08-17T17:26:02.522337+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:26:02.522337+0000","last_undegraded":"2024-08-17T17:26:02.522337+0000","last_fullsized":"2024-08-17T17:26:02.522337+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":79,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.528452+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:26:02.528452+0000","last_peered":"2024-08-17T17:26:02.528452+0000","last_clean":"2024-08-17T17:26:02.528452+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:26:02.528452+0000","last_undegraded":"2024-08-17T17:26:02.528452+0000","last_fullsized":"2024-08-17T17:26:02.528452+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":16,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.974650+0000","last_change":"2024-08-17T17:26:02.540241+0000","last_active":"2024-08-17T17:26:07.974650+0000","last_peered":"2024-08-17T17:26:07.974650+0000","last_clean":"2024-08-17T17:26:07.974650+0000","last_became_active":"2024-08-17T17:26:02.539822+0000","last_became_peered":"2024-08-17T17:26:02.539822+0000","last_unstale":"2024-08-17T17:26:07.974650+0000","last_undegraded":"2024-08-17T17:26:07.974650+0000","last_fullsized":"2024-08-17T17:26:07.974650+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:44.104335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":160,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937482+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:26:07.937482+0000","last_peered":"2024-08-17T17:26:07.937482+0000","last_clean":"2024-08-17T17:26:07.937482+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:26:07.937482+0000","last_undegraded":"2024-08-17T17:26:07.937482+0000","last_fullsized":"2024-08-17T17:26:07.937482+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":477,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.530499+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:26:02.530499+0000","last_peered":"2024-08-17T17:26:02.530499+0000","last_clean":"2024-08-17T17:26:02.530499+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:26:02.530499+0000","last_undegraded":"2024-08-17T17:26:02.530499+0000","last_fullsized":"2024-08-17T17:26:02.530499+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":183,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.527565+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:26:02.527565+0000","last_peered":"2024-08-17T17:26:02.527565+0000","last_clean":"2024-08-17T17:26:02.527565+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:26:02.527565+0000","last_undegraded":"2024-08-17T17:26:02.527565+0000","last_fullsized":"2024-08-17T17:26:02.527565+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":11,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.537701+0000","last_change":"2024-08-17T17:26:02.536369+0000","last_active":"2024-08-17T17:26:02.537701+0000","last_peered":"2024-08-17T17:26:02.537701+0000","last_clean":"2024-08-17T17:26:02.537701+0000","last_became_active":"2024-08-17T17:26:02.536022+0000","last_became_peered":"2024-08-17T17:26:02.536022+0000","last_unstale":"2024-08-17T17:26:02.537701+0000","last_undegraded":"2024-08-17T17:26:02.537701+0000","last_fullsized":"2024-08-17T17:26:02.537701+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:02.485847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":180,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.530366+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:26:02.530366+0000","last_peered":"2024-08-17T17:26:02.530366+0000","last_clean":"2024-08-17T17:26:02.530366+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:26:02.530366+0000","last_undegraded":"2024-08-17T17:26:02.530366+0000","last_fullsized":"2024-08-17T17:26:02.530366+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":731,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.527009+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:26:02.527009+0000","last_peered":"2024-08-17T17:26:02.527009+0000","last_clean":"2024-08-17T17:26:02.527009+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:26:02.527009+0000","last_undegraded":"2024-08-17T17:26:02.527009+0000","last_fullsized":"2024-08-17T17:26:02.527009+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":183,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975325+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:26:07.975325+0000","last_peered":"2024-08-17T17:26:07.975325+0000","last_clean":"2024-08-17T17:26:07.975325+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:26:07.975325+0000","last_undegraded":"2024-08-17T17:26:07.975325+0000","last_fullsized":"2024-08-17T17:26:07.975325+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:04.248364+0000","last_change":"2024-08-17T17:26:02.538931+0000","last_active":"2024-08-17T17:26:04.248364+0000","last_peered":"2024-08-17T17:26:04.248364+0000","last_clean":"2024-08-17T17:26:04.248364+0000","last_became_active":"2024-08-17T17:26:02.538728+0000","last_became_peered":"2024-08-17T17:26:02.538728+0000","last_unstale":"2024-08-17T17:26:04.248364+0000","last_undegraded":"2024-08-17T17:26:04.248364+0000","last_fullsized":"2024-08-17T17:26:04.248364+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:20.824450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":194,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937727+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:26:07.937727+0000","last_peered":"2024-08-17T17:26:07.937727+0000","last_clean":"2024-08-17T17:26:07.937727+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:26:07.937727+0000","last_undegraded":"2024-08-17T17:26:07.937727+0000","last_fullsized":"2024-08-17T17:26:07.937727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":697,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.529499+0000","last_change":"2024-08-17T17:25:52.491743+0000","last_active":"2024-08-17T17:26:02.529499+0000","last_peered":"2024-08-17T17:26:02.529499+0000","last_clean":"2024-08-17T17:26:02.529499+0000","last_became_active":"2024-08-17T17:25:52.491147+0000","last_became_peered":"2024-08-17T17:25:52.491147+0000","last_unstale":"2024-08-17T17:26:02.529499+0000","last_undegraded":"2024-08-17T17:26:02.529499+0000","last_fullsized":"2024-08-17T17:26:02.529499+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:20.776398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.936826+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:26:07.936826+0000","last_peered":"2024-08-17T17:26:07.936826+0000","last_clean":"2024-08-17T17:26:07.936826+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:26:07.936826+0000","last_undegraded":"2024-08-17T17:26:07.936826+0000","last_fullsized":"2024-08-17T17:26:07.936826+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":11,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:04.248246+0000","last_change":"2024-08-17T17:26:02.531249+0000","last_active":"2024-08-17T17:26:04.248246+0000","last_peered":"2024-08-17T17:26:04.248246+0000","last_clean":"2024-08-17T17:26:04.248246+0000","last_became_active":"2024-08-17T17:26:02.530950+0000","last_became_peered":"2024-08-17T17:26:02.530950+0000","last_unstale":"2024-08-17T17:26:04.248246+0000","last_undegraded":"2024-08-17T17:26:04.248246+0000","last_fullsized":"2024-08-17T17:26:04.248246+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:20.431512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":213,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975474+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:26:07.975474+0000","last_peered":"2024-08-17T17:26:07.975474+0000","last_clean":"2024-08-17T17:26:07.975474+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:26:07.975474+0000","last_undegraded":"2024-08-17T17:26:07.975474+0000","last_fullsized":"2024-08-17T17:26:07.975474+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":413,"reported_epoch":169,"state":"active+clean","last_fresh":"2024-08-17T17:26:03.888959+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:26:03.888959+0000","last_peered":"2024-08-17T17:26:03.888959+0000","last_clean":"2024-08-17T17:26:03.888959+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:26:03.888959+0000","last_undegraded":"2024-08-17T17:26:03.888959+0000","last_fullsized":"2024-08-17T17:26:03.888959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":62,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522077+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:26:02.522077+0000","last_peered":"2024-08-17T17:26:02.522077+0000","last_clean":"2024-08-17T17:26:02.522077+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:26:02.522077+0000","last_undegraded":"2024-08-17T17:26:02.522077+0000","last_fullsized":"2024-08-17T17:26:02.522077+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522411+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:26:02.522411+0000","last_peered":"2024-08-17T17:26:02.522411+0000","last_clean":"2024-08-17T17:26:02.522411+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:26:02.522411+0000","last_undegraded":"2024-08-17T17:26:02.522411+0000","last_fullsized":"2024-08-17T17:26:02.522411+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":63,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522256+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:26:02.522256+0000","last_peered":"2024-08-17T17:26:02.522256+0000","last_clean":"2024-08-17T17:26:02.522256+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:26:02.522256+0000","last_undegraded":"2024-08-17T17:26:02.522256+0000","last_fullsized":"2024-08-17T17:26:02.522256+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":173,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526698+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:26:02.526698+0000","last_peered":"2024-08-17T17:26:02.526698+0000","last_clean":"2024-08-17T17:26:02.526698+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:26:02.526698+0000","last_undegraded":"2024-08-17T17:26:02.526698+0000","last_fullsized":"2024-08-17T17:26:02.526698+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":164,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526777+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:26:02.526777+0000","last_peered":"2024-08-17T17:26:02.526777+0000","last_clean":"2024-08-17T17:26:02.526777+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:26:02.526777+0000","last_undegraded":"2024-08-17T17:26:02.526777+0000","last_fullsized":"2024-08-17T17:26:02.526777+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526297+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:26:02.526297+0000","last_peered":"2024-08-17T17:26:02.526297+0000","last_clean":"2024-08-17T17:26:02.526297+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:26:02.526297+0000","last_undegraded":"2024-08-17T17:26:02.526297+0000","last_fullsized":"2024-08-17T17:26:02.526297+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.936707+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:26:07.936707+0000","last_peered":"2024-08-17T17:26:07.936707+0000","last_clean":"2024-08-17T17:26:07.936707+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:26:07.936707+0000","last_undegraded":"2024-08-17T17:26:07.936707+0000","last_fullsized":"2024-08-17T17:26:07.936707+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975654+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:26:07.975654+0000","last_peered":"2024-08-17T17:26:07.975654+0000","last_clean":"2024-08-17T17:26:07.975654+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:26:07.975654+0000","last_undegraded":"2024-08-17T17:26:07.975654+0000","last_fullsized":"2024-08-17T17:26:07.975654+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.936821+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:26:07.936821+0000","last_peered":"2024-08-17T17:26:07.936821+0000","last_clean":"2024-08-17T17:26:07.936821+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:26:07.936821+0000","last_undegraded":"2024-08-17T17:26:07.936821+0000","last_fullsized":"2024-08-17T17:26:07.936821+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":140,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.526848+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:26:02.526848+0000","last_peered":"2024-08-17T17:26:02.526848+0000","last_clean":"2024-08-17T17:26:02.526848+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:26:02.526848+0000","last_undegraded":"2024-08-17T17:26:02.526848+0000","last_fullsized":"2024-08-17T17:26:02.526848+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":211,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937015+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:26:07.937015+0000","last_peered":"2024-08-17T17:26:07.937015+0000","last_clean":"2024-08-17T17:26:07.937015+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:26:07.937015+0000","last_undegraded":"2024-08-17T17:26:07.937015+0000","last_fullsized":"2024-08-17T17:26:07.937015+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":187,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975538+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:26:07.975538+0000","last_peered":"2024-08-17T17:26:07.975538+0000","last_clean":"2024-08-17T17:26:07.975538+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:26:07.975538+0000","last_undegraded":"2024-08-17T17:26:07.975538+0000","last_fullsized":"2024-08-17T17:26:07.975538+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":187,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.527102+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:26:02.527102+0000","last_peered":"2024-08-17T17:26:02.527102+0000","last_clean":"2024-08-17T17:26:02.527102+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:26:02.527102+0000","last_undegraded":"2024-08-17T17:26:02.527102+0000","last_fullsized":"2024-08-17T17:26:02.527102+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":207,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937364+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:26:07.937364+0000","last_peered":"2024-08-17T17:26:07.937364+0000","last_clean":"2024-08-17T17:26:07.937364+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:26:07.937364+0000","last_undegraded":"2024-08-17T17:26:07.937364+0000","last_fullsized":"2024-08-17T17:26:07.937364+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522254+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:26:02.522254+0000","last_peered":"2024-08-17T17:26:02.522254+0000","last_clean":"2024-08-17T17:26:02.522254+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:26:02.522254+0000","last_undegraded":"2024-08-17T17:26:02.522254+0000","last_fullsized":"2024-08-17T17:26:02.522254+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":216,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975117+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:26:07.975117+0000","last_peered":"2024-08-17T17:26:07.975117+0000","last_clean":"2024-08-17T17:26:07.975117+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:26:07.975117+0000","last_undegraded":"2024-08-17T17:26:07.975117+0000","last_fullsized":"2024-08-17T17:26:07.975117+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":193,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975268+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:26:07.975268+0000","last_peered":"2024-08-17T17:26:07.975268+0000","last_clean":"2024-08-17T17:26:07.975268+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:26:07.975268+0000","last_undegraded":"2024-08-17T17:26:07.975268+0000","last_fullsized":"2024-08-17T17:26:07.975268+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522267+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:26:02.522267+0000","last_peered":"2024-08-17T17:26:02.522267+0000","last_clean":"2024-08-17T17:26:02.522267+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:26:02.522267+0000","last_undegraded":"2024-08-17T17:26:02.522267+0000","last_fullsized":"2024-08-17T17:26:02.522267+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":154,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522233+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:26:02.522233+0000","last_peered":"2024-08-17T17:26:02.522233+0000","last_clean":"2024-08-17T17:26:02.522233+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:26:02.522233+0000","last_undegraded":"2024-08-17T17:26:02.522233+0000","last_fullsized":"2024-08-17T17:26:02.522233+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522167+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:26:02.522167+0000","last_peered":"2024-08-17T17:26:02.522167+0000","last_clean":"2024-08-17T17:26:02.522167+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:26:02.522167+0000","last_undegraded":"2024-08-17T17:26:02.522167+0000","last_fullsized":"2024-08-17T17:26:02.522167+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":139,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.522462+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:26:02.522462+0000","last_peered":"2024-08-17T17:26:02.522462+0000","last_clean":"2024-08-17T17:26:02.522462+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:26:02.522462+0000","last_undegraded":"2024-08-17T17:26:02.522462+0000","last_fullsized":"2024-08-17T17:26:02.522462+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937025+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:26:07.937025+0000","last_peered":"2024-08-17T17:26:07.937025+0000","last_clean":"2024-08-17T17:26:07.937025+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:26:07.937025+0000","last_undegraded":"2024-08-17T17:26:07.937025+0000","last_fullsized":"2024-08-17T17:26:07.937025+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":201,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.937592+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:26:07.937592+0000","last_peered":"2024-08-17T17:26:07.937592+0000","last_clean":"2024-08-17T17:26:07.937592+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:26:07.937592+0000","last_undegraded":"2024-08-17T17:26:07.937592+0000","last_fullsized":"2024-08-17T17:26:07.937592+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":168,"state":"active+clean","last_fresh":"2024-08-17T17:26:02.521970+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:26:02.521970+0000","last_peered":"2024-08-17T17:26:02.521970+0000","last_clean":"2024-08-17T17:26:02.521970+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:26:02.521970+0000","last_undegraded":"2024-08-17T17:26:02.521970+0000","last_fullsized":"2024-08-17T17:26:02.521970+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":204,"reported_epoch":170,"state":"active+clean","last_fresh":"2024-08-17T17:26:07.975549+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:26:07.975549+0000","last_peered":"2024-08-17T17:26:07.975549+0000","last_clean":"2024-08-17T17:26:07.975549+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:26:07.975549+0000","last_undegraded":"2024-08-17T17:26:07.975549+0000","last_fullsized":"2024-08-17T17:26:07.975549+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201551,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":158420,"kb_used_data":6728,"kb_used_omap":1,"kb_used_meta":151678,"kb_avail":93586732,"statfs":{"total":95995035648,"available":95832813568,"internally_reserved":0,"allocated":6889472,"data_stored":5915345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":155318731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463189,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105936,"kb_used_data":2624,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":93639216,"statfs":{"total":95995035648,"available":95886557184,"internally_reserved":0,"allocated":2686976,"data_stored":1721231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724825,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986461,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":221840,"kb_used_data":6144,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93523312,"statfs":{"total":95995035648,"available":95767871488,"internally_reserved":0,"allocated":6291456,"data_stored":5324915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":220854726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435353,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123964,"kb_used_data":2032,"kb_used_omap":5,"kb_used_meta":121914,"kb_avail":93621188,"statfs":{"total":95995035648,"available":95868096512,"internally_reserved":0,"allocated":2080768,"data_stored":1125881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":124840009},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598879,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":200316,"kb_used_data":2032,"kb_used_omap":7,"kb_used_meta":198264,"kb_avail":93544836,"statfs":{"total":95995035648,"available":95789912064,"internally_reserved":0,"allocated":2080768,"data_stored":1126337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":203022509},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762405,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":137832,"kb_used_data":2012,"kb_used_omap":7,"kb_used_meta":135800,"kb_avail":93607320,"statfs":{"total":95995035648,"available":95853895680,"internally_reserved":0,"allocated":2060288,"data_stored":1116954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":139059617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925932,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245264,"kb_used_data":6716,"kb_used_omap":7,"kb_used_meta":238520,"kb_avail":93499888,"statfs":{"total":95995035648,"available":95743885312,"internally_reserved":0,"allocated":6877184,"data_stored":5911289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":244245084},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:26:12.187 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:26:12.187 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:26:12.188 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:12.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:12.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:12.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:12.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:12.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:12.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:12.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:12.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:12.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:12.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:12.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:12.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:12 smithi175 ceph-mon[101434]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:12.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:12 smithi064 ceph-mon[104516]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:12.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:12 smithi064 ceph-mon[109246]: pgmap v526: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:12.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:12.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:13.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:13.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:13.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:13.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:13.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:13.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:13.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:13.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:13.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:13.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:13.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:13 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:13.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:13 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:13.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:13 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:13.951 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-08-17T17:26:13.952 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-08-17T17:26:13.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:13.954 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:26:13.954 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:26:13.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:26:14.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:26:14.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_185391 2024-08-17T17:26:14.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:26:14.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:26:14.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.114 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:14.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:14.183 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:14.184 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:14.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:13] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:26:14.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:14.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:14.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:14.607 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:14.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:14.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:14.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:14.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:14.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:14 smithi175 ceph-mon[101434]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:14.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:14 smithi064 ceph-mon[104516]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:14 smithi064 ceph-mon[109246]: pgmap v528: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:14.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:15.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3234 3218 201.114 201.125 0.00662338 0.00495705 2024-08-17T17:26:15.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:15.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.275 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:15.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:15.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:15.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:15.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:26:15.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:15.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:15.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.862 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:15.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:15 smithi064 ceph-mon[104516]: pgmap v529: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:15.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:15 smithi064 ceph-mon[109246]: pgmap v529: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:15.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:15.992 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:16.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6260 6244 195.111 189.125 0.00826327 0.0051143 2024-08-17T17:26:16.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.136 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:16.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:16.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:15 smithi175 ceph-mon[101434]: pgmap v529: 89 pgs: 89 active+clean; 4.6 MiB data, 1.1 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:26:16.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:16.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:16.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:16.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:16.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:16.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:16.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:16.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:26:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:16.595 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:26:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:16.635 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:26:16.635 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:16.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:16.674 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:26:16.674 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:16.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:16.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:16.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:26:16.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:16.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:26:16.746 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:16.746 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3360206133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:16.746 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3360206133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:16.779 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:16.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:26:16.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:16.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:16.823 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:26:16.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:16.909 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:17.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9026 9010 187.696 172.875 0.00572325 0.00531933 2024-08-17T17:26:17.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:17.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3360206133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:17.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:17.223 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:17.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:17.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:17.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:17.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:17.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:17.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:17.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:17.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:17.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:17.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:17.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:17.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:17.909 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:17 smithi064 ceph-mon[104516]: pgmap v530: 89 pgs: 89 active+clean; 155 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s 2024-08-17T17:26:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:17 smithi064 ceph-mon[109246]: pgmap v530: 89 pgs: 89 active+clean; 155 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s 2024-08-17T17:26:17.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:18.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 11840 11824 184.736 175.875 0.00474635 0.00537499 2024-08-17T17:26:18.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:18.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:18.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:17 smithi175 ceph-mon[101434]: pgmap v530: 89 pgs: 89 active+clean; 155 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s 2024-08-17T17:26:18.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:18.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:18.304 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:18.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:18.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.537 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:18.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:18.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:18.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:18.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:18.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:18.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:18.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:19.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 14458 14442 180.511 163.625 0.00324158 0.00553181 2024-08-17T17:26:19.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:19.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:19.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:19.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:19.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:19.408 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:19.510 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:19.535 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:19.636 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:19.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:19.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:19.783 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:19.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:19.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:19.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.920 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:19.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:20.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 16690 16674 173.674 139.5 0.00713163 0.00575132 2024-08-17T17:26:20.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.062 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:19 smithi064 ceph-mon[104516]: pgmap v531: 89 pgs: 89 active+clean; 524 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2024-08-17T17:26:20.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:19 smithi064 ceph-mon[109246]: pgmap v531: 89 pgs: 89 active+clean; 524 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2024-08-17T17:26:20.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:20.191 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:20.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:20.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:20.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:19 smithi175 ceph-mon[101434]: pgmap v531: 89 pgs: 89 active+clean; 524 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s 2024-08-17T17:26:20.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:20.452 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:20.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:20.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:20.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:20.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:20.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:20.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:20.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:20.956 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:26:20.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:20.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:21.011 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 18942 18926 168.968 140.75 0.0107431 0.00590984 2024-08-17T17:26:21.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:21.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:20 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/495069567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:20 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:20 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/495069567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:20 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.132 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:21.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:21.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:21.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:21.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.386 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:20 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/495069567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:20 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:21.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:21.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:21.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:21.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:21.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:21.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.915 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:21.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:22.011 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 20962 20946 163.627 126.25 0.00801499 0.00610508 2024-08-17T17:26:22.016 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:22.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:22.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:22.176 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:22.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:22.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:21 smithi175 ceph-mon[101434]: pgmap v532: 89 pgs: 89 active+clean; 744 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:26:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:21 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:21 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:26:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:21 smithi175 ceph-mon[101434]: osdmap e172: 8 total, 8 up, 7 in 2024-08-17T17:26:22.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[104516]: pgmap v532: 89 pgs: 89 active+clean; 744 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:26:22.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[104516]: osdmap e172: 8 total, 8 up, 7 in 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[109246]: pgmap v532: 89 pgs: 89 active+clean; 744 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:22.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:26:22.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:22.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:21 smithi064 ceph-mon[109246]: osdmap e172: 8 total, 8 up, 7 in 2024-08-17T17:26:22.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:22.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:22.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:22.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:23.011 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 23215 23199 161.09 140.812 0.00437672 0.00620116 2024-08-17T17:26:23.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:23.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:23.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:23.259 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:23.404 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:23.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:23.528 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:23.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:23.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:23.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:23.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:23.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:24.011 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 25351 25335 158.33 133.5 0.00237344 0.00630914 2024-08-17T17:26:24.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:24.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:24.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:24.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:24.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:24.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:24.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:23 smithi175 ceph-mon[101434]: pgmap v534: 89 pgs: 89 active+clean; 985 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:26:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:23 smithi064 ceph-mon[104516]: pgmap v534: 89 pgs: 89 active+clean; 985 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:26:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:23 smithi064 ceph-mon[109246]: pgmap v534: 89 pgs: 89 active+clean; 985 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:26:24.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:23] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:26:24.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:24.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:24.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:24.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:24.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:24.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:24.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:24.896 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:24.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:25.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 27222 27206 154.549 116.938 0.0084702 0.0064594 2024-08-17T17:26:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:25.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:25.140 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:25.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:25.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:25.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:25.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:25.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:25.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-17T17:26:25.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:26:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-17T17:26:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:26:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-17T17:26:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:26:25.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:25.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:25.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:25.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:25.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:25.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:25.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:25.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:25.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:25.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:26.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 29012 28996 150.992 111.875 0.0109194 0.00661444 2024-08-17T17:26:26.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:26.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:26.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:26.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: pgmap v535: 89 pgs: 89 active+clean; 1.4 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: osdmap e173: 8 total, 8 up, 7 in 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:26.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:26.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: pgmap v535: 89 pgs: 89 active+clean; 1.4 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:26:26.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-17T17:26:26.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:26:26.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: osdmap e173: 8 total, 8 up, 7 in 2024-08-17T17:26:26.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:26.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:26.312 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: pgmap v535: 89 pgs: 89 active+clean; 1.4 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:26:26.312 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-17T17:26:26.312 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:26:26.313 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: osdmap e173: 8 total, 8 up, 7 in 2024-08-17T17:26:26.313 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:26.313 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:26.337 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:26.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:26.467 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:26.497 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:26.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:26.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:26.743 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.769 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:26.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:26.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:26.920 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:27.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 30892 30876 148.415 117.5 0.00959479 0.0067309 2024-08-17T17:26:27.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:27.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:27.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:27.190 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:27.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:27.320 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:27.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:27.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:26 smithi064 ceph-mon[104516]: osdmap e174: 8 total, 8 up, 7 in 2024-08-17T17:26:27.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:26 smithi064 ceph-mon[109246]: osdmap e174: 8 total, 8 up, 7 in 2024-08-17T17:26:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:26 smithi175 ceph-mon[101434]: osdmap e174: 8 total, 8 up, 7 in 2024-08-17T17:26:27.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:27.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:27.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:27.595 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:27.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:27.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:26:27.750 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:26:27.754 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:26:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:26:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:26:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:26:27.755 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:26:27.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:27.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:27.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 32592 32576 145.403 106.25 0.0101643 0.0068713 2024-08-17T17:26:28.112 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:28.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:28.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:26:28.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:28.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:28 smithi064 ceph-mon[109246]: pgmap v538: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:26:28.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:28 smithi064 ceph-mon[104516]: pgmap v538: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:26:28.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:28 smithi175 ceph-mon[101434]: pgmap v538: 89 pgs: 1 peering, 88 active+clean; 1.6 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:26:28.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:28.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:26:28.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:28.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:29.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 34454 34438 143.467 116.375 0.0088061 0.00696387 2024-08-17T17:26:29.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:29 smithi064 ceph-mon[104516]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:29 smithi064 ceph-mon[109246]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:29.298 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:29.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.339 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":538,"stamp":"2024-08-17T17:26:26.883503+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1730615269,"num_objects":26362,"num_object_clones":0,"num_object_copies":79086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26362,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":27457,"num_write_kb":1700417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":10165431,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27516,"ondisk_log_size":27516,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6121672,"kb_used_data":4831608,"kb_used_omap":33,"kb_used_meta":1289950,"kb_avail":650094392,"statfs":{"total":671965249536,"available":665696657408,"internally_reserved":0,"allocated":4947566592,"data_stored":4940738343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1320909387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,5,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":950337536,"num_objects":14501,"num_object_clones":0,"num_object_copies":43503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14501,"num_write_kb":928064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001536"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963484+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:26:20.963484+0000","last_peered":"2024-08-17T17:26:20.963484+0000","last_clean":"2024-08-17T17:26:20.963484+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:26:20.963484+0000","last_undegraded":"2024-08-17T17:26:20.963484+0000","last_fullsized":"2024-08-17T17:26:20.963484+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":167,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950656+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:26:20.950656+0000","last_peered":"2024-08-17T17:26:20.950656+0000","last_clean":"2024-08-17T17:26:20.950656+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:26:20.950656+0000","last_undegraded":"2024-08-17T17:26:20.950656+0000","last_fullsized":"2024-08-17T17:26:20.950656+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950449+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:26:20.950449+0000","last_peered":"2024-08-17T17:26:20.950449+0000","last_clean":"2024-08-17T17:26:20.950449+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:26:20.950449+0000","last_undegraded":"2024-08-17T17:26:20.950449+0000","last_fullsized":"2024-08-17T17:26:20.950449+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":190,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947138+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:26:20.947138+0000","last_peered":"2024-08-17T17:26:20.947138+0000","last_clean":"2024-08-17T17:26:20.947138+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:26:20.947138+0000","last_undegraded":"2024-08-17T17:26:20.947138+0000","last_fullsized":"2024-08-17T17:26:20.947138+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":190,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947483+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:26:20.947483+0000","last_peered":"2024-08-17T17:26:20.947483+0000","last_clean":"2024-08-17T17:26:20.947483+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:26:20.947483+0000","last_undegraded":"2024-08-17T17:26:20.947483+0000","last_fullsized":"2024-08-17T17:26:20.947483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":138,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950313+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:26:20.950313+0000","last_peered":"2024-08-17T17:26:20.950313+0000","last_clean":"2024-08-17T17:26:20.950313+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:26:20.950313+0000","last_undegraded":"2024-08-17T17:26:20.950313+0000","last_fullsized":"2024-08-17T17:26:20.950313+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":199,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963766+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:26:20.963766+0000","last_peered":"2024-08-17T17:26:20.963766+0000","last_clean":"2024-08-17T17:26:20.963766+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:26:20.963766+0000","last_undegraded":"2024-08-17T17:26:20.963766+0000","last_fullsized":"2024-08-17T17:26:20.963766+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":162,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952272+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:26:20.952272+0000","last_peered":"2024-08-17T17:26:20.952272+0000","last_clean":"2024-08-17T17:26:20.952272+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:26:20.952272+0000","last_undegraded":"2024-08-17T17:26:20.952272+0000","last_fullsized":"2024-08-17T17:26:20.952272+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"173'1841","reported_seq":1855,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.606601+0000","last_change":"2024-08-17T17:26:02.534643+0000","last_active":"2024-08-17T17:26:25.606601+0000","last_peered":"2024-08-17T17:26:25.606601+0000","last_clean":"2024-08-17T17:26:25.606601+0000","last_became_active":"2024-08-17T17:26:02.534394+0000","last_became_peered":"2024-08-17T17:26:02.534394+0000","last_unstale":"2024-08-17T17:26:25.606601+0000","last_undegraded":"2024-08-17T17:26:25.606601+0000","last_fullsized":"2024-08-17T17:26:25.606601+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1841,"log_dups_size":0,"ondisk_log_size":1841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:29.181237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120651776,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1841,"num_write_kb":117824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":163,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950635+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:26:20.950635+0000","last_peered":"2024-08-17T17:26:20.950635+0000","last_clean":"2024-08-17T17:26:20.950635+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:26:20.950635+0000","last_undegraded":"2024-08-17T17:26:20.950635+0000","last_fullsized":"2024-08-17T17:26:20.950635+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950226+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:26:20.950226+0000","last_peered":"2024-08-17T17:26:20.950226+0000","last_clean":"2024-08-17T17:26:20.950226+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:26:20.950226+0000","last_undegraded":"2024-08-17T17:26:20.950226+0000","last_fullsized":"2024-08-17T17:26:20.950226+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"172'1553","reported_seq":1574,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.774999+0000","last_change":"2024-08-17T17:26:02.539050+0000","last_active":"2024-08-17T17:26:23.774999+0000","last_peered":"2024-08-17T17:26:23.774999+0000","last_clean":"2024-08-17T17:26:23.774999+0000","last_became_active":"2024-08-17T17:26:02.538574+0000","last_became_peered":"2024-08-17T17:26:02.538574+0000","last_unstale":"2024-08-17T17:26:23.774999+0000","last_undegraded":"2024-08-17T17:26:23.774999+0000","last_fullsized":"2024-08-17T17:26:23.774999+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1553,"log_dups_size":0,"ondisk_log_size":1553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:15.744543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101777408,"num_objects":1553,"num_object_clones":0,"num_object_copies":4659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1553,"num_write_kb":99392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":164,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950753+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:26:20.950753+0000","last_peered":"2024-08-17T17:26:20.950753+0000","last_clean":"2024-08-17T17:26:20.950753+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:26:20.950753+0000","last_undegraded":"2024-08-17T17:26:20.950753+0000","last_fullsized":"2024-08-17T17:26:20.950753+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.982969+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:26:24.982969+0000","last_peered":"2024-08-17T17:26:24.982969+0000","last_clean":"2024-08-17T17:26:24.982969+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:26:24.982969+0000","last_undegraded":"2024-08-17T17:26:24.982969+0000","last_fullsized":"2024-08-17T17:26:24.982969+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"173'1776","reported_seq":1791,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.985269+0000","last_change":"2024-08-17T17:26:02.538436+0000","last_active":"2024-08-17T17:26:25.985269+0000","last_peered":"2024-08-17T17:26:25.985269+0000","last_clean":"2024-08-17T17:26:25.985269+0000","last_became_active":"2024-08-17T17:26:02.538195+0000","last_became_peered":"2024-08-17T17:26:02.538195+0000","last_unstale":"2024-08-17T17:26:25.985269+0000","last_undegraded":"2024-08-17T17:26:25.985269+0000","last_fullsized":"2024-08-17T17:26:25.985269+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1776,"log_dups_size":0,"ondisk_log_size":1776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:29.623332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116391936,"num_objects":1776,"num_object_clones":0,"num_object_copies":5328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1776,"num_write_kb":113664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":135,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950619+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:26:20.950619+0000","last_peered":"2024-08-17T17:26:20.950619+0000","last_clean":"2024-08-17T17:26:20.950619+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:26:20.950619+0000","last_undegraded":"2024-08-17T17:26:20.950619+0000","last_fullsized":"2024-08-17T17:26:20.950619+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":173,"state":"peering","last_fresh":"2024-08-17T17:26:25.003795+0000","last_change":"2024-08-17T17:26:25.004552+0000","last_active":"2024-08-17T17:26:25.003795+0000","last_peered":"2024-08-17T17:26:20.952601+0000","last_clean":"2024-08-17T17:26:20.952601+0000","last_became_active":"2024-08-17T17:17:50.474625+0000","last_became_peered":"2024-08-17T17:17:50.474625+0000","last_unstale":"2024-08-17T17:26:25.003795+0000","last_undegraded":"2024-08-17T17:26:25.003795+0000","last_fullsized":"2024-08-17T17:26:25.003795+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:42:09.092585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"172'1470","reported_seq":1491,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.435109+0000","last_change":"2024-08-17T17:26:02.538250+0000","last_active":"2024-08-17T17:26:23.435109+0000","last_peered":"2024-08-17T17:26:23.435109+0000","last_clean":"2024-08-17T17:26:23.435109+0000","last_became_active":"2024-08-17T17:26:02.537877+0000","last_became_peered":"2024-08-17T17:26:02.537877+0000","last_unstale":"2024-08-17T17:26:23.435109+0000","last_undegraded":"2024-08-17T17:26:23.435109+0000","last_fullsized":"2024-08-17T17:26:23.435109+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1470,"log_dups_size":0,"ondisk_log_size":1470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:07.927741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96337920,"num_objects":1470,"num_object_clones":0,"num_object_copies":4410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1470,"num_write_kb":94080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":148,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.964230+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:26:20.964230+0000","last_peered":"2024-08-17T17:26:20.964230+0000","last_clean":"2024-08-17T17:26:20.964230+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:26:20.964230+0000","last_undegraded":"2024-08-17T17:26:20.964230+0000","last_fullsized":"2024-08-17T17:26:20.964230+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":205,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947429+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:26:20.947429+0000","last_peered":"2024-08-17T17:26:20.947429+0000","last_clean":"2024-08-17T17:26:20.947429+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:26:20.947429+0000","last_undegraded":"2024-08-17T17:26:20.947429+0000","last_fullsized":"2024-08-17T17:26:20.947429+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"174'1860","reported_seq":1876,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-08-17T17:26:26.515935+0000","last_change":"2024-08-17T17:26:02.533130+0000","last_active":"2024-08-17T17:26:26.515935+0000","last_peered":"2024-08-17T17:26:26.515935+0000","last_clean":"2024-08-17T17:26:26.515935+0000","last_became_active":"2024-08-17T17:26:02.532783+0000","last_became_peered":"2024-08-17T17:26:02.532783+0000","last_unstale":"2024-08-17T17:26:26.515935+0000","last_undegraded":"2024-08-17T17:26:26.515935+0000","last_fullsized":"2024-08-17T17:26:26.515935+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1860,"log_dups_size":0,"ondisk_log_size":1860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:32.674058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1860,"num_write_kb":119040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":220,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963228+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:26:20.963228+0000","last_peered":"2024-08-17T17:26:20.963228+0000","last_clean":"2024-08-17T17:26:20.963228+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:26:20.963228+0000","last_undegraded":"2024-08-17T17:26:20.963228+0000","last_fullsized":"2024-08-17T17:26:20.963228+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983240+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:26:24.983240+0000","last_peered":"2024-08-17T17:26:24.983240+0000","last_clean":"2024-08-17T17:26:24.983240+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:26:24.983240+0000","last_undegraded":"2024-08-17T17:26:24.983240+0000","last_fullsized":"2024-08-17T17:26:24.983240+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"172'1497","reported_seq":1517,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.032581+0000","last_change":"2024-08-17T17:26:02.537749+0000","last_active":"2024-08-17T17:26:23.032581+0000","last_peered":"2024-08-17T17:26:23.032581+0000","last_clean":"2024-08-17T17:26:23.032581+0000","last_became_active":"2024-08-17T17:26:02.537124+0000","last_became_peered":"2024-08-17T17:26:02.537124+0000","last_unstale":"2024-08-17T17:26:23.032581+0000","last_undegraded":"2024-08-17T17:26:23.032581+0000","last_fullsized":"2024-08-17T17:26:23.032581+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1497,"log_dups_size":0,"ondisk_log_size":1497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:53.513054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98107392,"num_objects":1497,"num_object_clones":0,"num_object_copies":4491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1497,"num_write_kb":95808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952687+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:26:20.952687+0000","last_peered":"2024-08-17T17:26:20.952687+0000","last_clean":"2024-08-17T17:26:20.952687+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:26:20.952687+0000","last_undegraded":"2024-08-17T17:26:20.952687+0000","last_fullsized":"2024-08-17T17:26:20.952687+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983341+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:26:24.983341+0000","last_peered":"2024-08-17T17:26:24.983341+0000","last_clean":"2024-08-17T17:26:24.983341+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:26:24.983341+0000","last_undegraded":"2024-08-17T17:26:24.983341+0000","last_fullsized":"2024-08-17T17:26:24.983341+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"172'1559","reported_seq":1580,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.740286+0000","last_change":"2024-08-17T17:26:02.538371+0000","last_active":"2024-08-17T17:26:23.740286+0000","last_peered":"2024-08-17T17:26:23.740286+0000","last_clean":"2024-08-17T17:26:23.740286+0000","last_became_active":"2024-08-17T17:26:02.537919+0000","last_became_peered":"2024-08-17T17:26:02.537919+0000","last_unstale":"2024-08-17T17:26:23.740286+0000","last_undegraded":"2024-08-17T17:26:23.740286+0000","last_fullsized":"2024-08-17T17:26:23.740286+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1559,"log_dups_size":0,"ondisk_log_size":1559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:58:21.757703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102170624,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1559,"num_write_kb":99776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.982877+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:26:24.982877+0000","last_peered":"2024-08-17T17:26:24.982877+0000","last_clean":"2024-08-17T17:26:24.982877+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:26:24.982877+0000","last_undegraded":"2024-08-17T17:26:24.982877+0000","last_fullsized":"2024-08-17T17:26:24.982877+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983406+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:26:24.983406+0000","last_peered":"2024-08-17T17:26:24.983406+0000","last_clean":"2024-08-17T17:26:24.983406+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:26:24.983406+0000","last_undegraded":"2024-08-17T17:26:24.983406+0000","last_fullsized":"2024-08-17T17:26:24.983406+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"172'1555","reported_seq":1575,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.946789+0000","last_change":"2024-08-17T17:26:02.536813+0000","last_active":"2024-08-17T17:26:23.946789+0000","last_peered":"2024-08-17T17:26:23.946789+0000","last_clean":"2024-08-17T17:26:23.946789+0000","last_became_active":"2024-08-17T17:26:02.535602+0000","last_became_peered":"2024-08-17T17:26:02.535602+0000","last_unstale":"2024-08-17T17:26:23.946789+0000","last_undegraded":"2024-08-17T17:26:23.946789+0000","last_fullsized":"2024-08-17T17:26:23.946789+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1555,"log_dups_size":0,"ondisk_log_size":1555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:41.209612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1555,"num_write_kb":99520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963324+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:26:20.963324+0000","last_peered":"2024-08-17T17:26:20.963324+0000","last_clean":"2024-08-17T17:26:20.963324+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:26:20.963324+0000","last_undegraded":"2024-08-17T17:26:20.963324+0000","last_fullsized":"2024-08-17T17:26:20.963324+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":187,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952820+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:26:20.952820+0000","last_peered":"2024-08-17T17:26:20.952820+0000","last_clean":"2024-08-17T17:26:20.952820+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:26:20.952820+0000","last_undegraded":"2024-08-17T17:26:20.952820+0000","last_fullsized":"2024-08-17T17:26:20.952820+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"172'1609","reported_seq":1630,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.774216+0000","last_change":"2024-08-17T17:26:02.536026+0000","last_active":"2024-08-17T17:26:23.774216+0000","last_peered":"2024-08-17T17:26:23.774216+0000","last_clean":"2024-08-17T17:26:23.774216+0000","last_became_active":"2024-08-17T17:26:02.535621+0000","last_became_peered":"2024-08-17T17:26:02.535621+0000","last_unstale":"2024-08-17T17:26:23.774216+0000","last_undegraded":"2024-08-17T17:26:23.774216+0000","last_fullsized":"2024-08-17T17:26:23.774216+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1609,"log_dups_size":0,"ondisk_log_size":1609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:22.518064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1609,"num_write_kb":102976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":468,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950424+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:26:20.950424+0000","last_peered":"2024-08-17T17:26:20.950424+0000","last_clean":"2024-08-17T17:26:20.950424+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:26:20.950424+0000","last_undegraded":"2024-08-17T17:26:20.950424+0000","last_fullsized":"2024-08-17T17:26:20.950424+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":715,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952417+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:26:20.952417+0000","last_peered":"2024-08-17T17:26:20.952417+0000","last_clean":"2024-08-17T17:26:20.952417+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:26:20.952417+0000","last_undegraded":"2024-08-17T17:26:20.952417+0000","last_fullsized":"2024-08-17T17:26:20.952417+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.962538+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:26:20.962538+0000","last_peered":"2024-08-17T17:26:20.962538+0000","last_clean":"2024-08-17T17:26:20.962538+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:26:20.962538+0000","last_undegraded":"2024-08-17T17:26:20.962538+0000","last_fullsized":"2024-08-17T17:26:20.962538+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":138,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952950+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:26:20.952950+0000","last_peered":"2024-08-17T17:26:20.952950+0000","last_clean":"2024-08-17T17:26:20.952950+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:26:20.952950+0000","last_undegraded":"2024-08-17T17:26:20.952950+0000","last_fullsized":"2024-08-17T17:26:20.952950+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"172'1592","reported_seq":1613,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.936033+0000","last_change":"2024-08-17T17:26:02.539873+0000","last_active":"2024-08-17T17:26:23.936033+0000","last_peered":"2024-08-17T17:26:23.936033+0000","last_clean":"2024-08-17T17:26:23.936033+0000","last_became_active":"2024-08-17T17:26:02.539492+0000","last_became_peered":"2024-08-17T17:26:02.539492+0000","last_unstale":"2024-08-17T17:26:23.936033+0000","last_undegraded":"2024-08-17T17:26:23.936033+0000","last_fullsized":"2024-08-17T17:26:23.936033+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1592,"log_dups_size":0,"ondisk_log_size":1592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:31:34.168274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104333312,"num_objects":1592,"num_object_clones":0,"num_object_copies":4776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1592,"num_write_kb":101888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":545,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950233+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:26:20.950233+0000","last_peered":"2024-08-17T17:26:20.950233+0000","last_clean":"2024-08-17T17:26:20.950233+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:26:20.950233+0000","last_undegraded":"2024-08-17T17:26:20.950233+0000","last_fullsized":"2024-08-17T17:26:20.950233+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":205,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947498+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:26:20.947498+0000","last_peered":"2024-08-17T17:26:20.947498+0000","last_clean":"2024-08-17T17:26:20.947498+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:26:20.947498+0000","last_undegraded":"2024-08-17T17:26:20.947498+0000","last_fullsized":"2024-08-17T17:26:20.947498+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":202,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947697+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:26:20.947697+0000","last_peered":"2024-08-17T17:26:20.947697+0000","last_clean":"2024-08-17T17:26:20.947697+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:26:20.947697+0000","last_undegraded":"2024-08-17T17:26:20.947697+0000","last_fullsized":"2024-08-17T17:26:20.947697+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"173'1816","reported_seq":1831,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.958984+0000","last_change":"2024-08-17T17:26:02.532201+0000","last_active":"2024-08-17T17:26:25.958984+0000","last_peered":"2024-08-17T17:26:25.958984+0000","last_clean":"2024-08-17T17:26:25.958984+0000","last_became_active":"2024-08-17T17:26:02.531942+0000","last_became_peered":"2024-08-17T17:26:02.531942+0000","last_unstale":"2024-08-17T17:26:25.958984+0000","last_undegraded":"2024-08-17T17:26:25.958984+0000","last_fullsized":"2024-08-17T17:26:25.958984+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1816,"log_dups_size":0,"ondisk_log_size":1816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:51.278560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1816,"num_write_kb":116224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":601,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952417+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:26:20.952417+0000","last_peered":"2024-08-17T17:26:20.952417+0000","last_clean":"2024-08-17T17:26:20.952417+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:26:20.952417+0000","last_undegraded":"2024-08-17T17:26:20.952417+0000","last_fullsized":"2024-08-17T17:26:20.952417+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":182,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.957328+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:26:20.957328+0000","last_peered":"2024-08-17T17:26:20.957328+0000","last_clean":"2024-08-17T17:26:20.957328+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:26:20.957328+0000","last_undegraded":"2024-08-17T17:26:20.957328+0000","last_fullsized":"2024-08-17T17:26:20.957328+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.957604+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:26:20.957604+0000","last_peered":"2024-08-17T17:26:20.957604+0000","last_clean":"2024-08-17T17:26:20.957604+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:26:20.957604+0000","last_undegraded":"2024-08-17T17:26:20.957604+0000","last_fullsized":"2024-08-17T17:26:20.957604+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"172'1555","reported_seq":1576,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.771852+0000","last_change":"2024-08-17T17:26:02.539405+0000","last_active":"2024-08-17T17:26:23.771852+0000","last_peered":"2024-08-17T17:26:23.771852+0000","last_clean":"2024-08-17T17:26:23.771852+0000","last_became_active":"2024-08-17T17:26:02.539076+0000","last_became_peered":"2024-08-17T17:26:02.539076+0000","last_unstale":"2024-08-17T17:26:23.771852+0000","last_undegraded":"2024-08-17T17:26:23.771852+0000","last_fullsized":"2024-08-17T17:26:23.771852+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1555,"log_dups_size":0,"ondisk_log_size":1555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:33:29.415967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1555,"num_write_kb":99520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1029,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.957510+0000","last_change":"2024-08-17T17:25:52.490672+0000","last_active":"2024-08-17T17:26:20.957510+0000","last_peered":"2024-08-17T17:26:20.957510+0000","last_clean":"2024-08-17T17:26:20.957510+0000","last_became_active":"2024-08-17T17:25:52.490180+0000","last_became_peered":"2024-08-17T17:25:52.490180+0000","last_unstale":"2024-08-17T17:26:20.957510+0000","last_undegraded":"2024-08-17T17:26:20.957510+0000","last_fullsized":"2024-08-17T17:26:20.957510+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:46.014207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":56,"reported_epoch":171,"state":"active+clean","last_fresh":"2024-08-17T17:26:11.214584+0000","last_change":"2024-08-17T17:22:21.209140+0000","last_active":"2024-08-17T17:26:11.214584+0000","last_peered":"2024-08-17T17:26:11.214584+0000","last_clean":"2024-08-17T17:26:11.214584+0000","last_became_active":"2024-08-17T17:22:21.208665+0000","last_became_peered":"2024-08-17T17:22:21.208665+0000","last_unstale":"2024-08-17T17:26:11.214584+0000","last_undegraded":"2024-08-17T17:26:11.214584+0000","last_fullsized":"2024-08-17T17:26:11.214584+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:37.919003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.958063+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:26:20.958063+0000","last_peered":"2024-08-17T17:26:20.958063+0000","last_clean":"2024-08-17T17:26:20.958063+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:26:20.958063+0000","last_undegraded":"2024-08-17T17:26:20.958063+0000","last_fullsized":"2024-08-17T17:26:20.958063+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"172'1496","reported_seq":1515,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.449590+0000","last_change":"2024-08-17T17:26:02.540241+0000","last_active":"2024-08-17T17:26:23.449590+0000","last_peered":"2024-08-17T17:26:23.449590+0000","last_clean":"2024-08-17T17:26:23.449590+0000","last_became_active":"2024-08-17T17:26:02.539822+0000","last_became_peered":"2024-08-17T17:26:02.539822+0000","last_unstale":"2024-08-17T17:26:23.449590+0000","last_undegraded":"2024-08-17T17:26:23.449590+0000","last_fullsized":"2024-08-17T17:26:23.449590+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1496,"log_dups_size":0,"ondisk_log_size":1496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:08.004216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98041856,"num_objects":1496,"num_object_clones":0,"num_object_copies":4488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1496,"num_write_kb":95744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":164,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.962935+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:26:20.962935+0000","last_peered":"2024-08-17T17:26:20.962935+0000","last_clean":"2024-08-17T17:26:20.962935+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:26:20.962935+0000","last_undegraded":"2024-08-17T17:26:20.962935+0000","last_fullsized":"2024-08-17T17:26:20.962935+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":486,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.958240+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:26:20.958240+0000","last_peered":"2024-08-17T17:26:20.958240+0000","last_clean":"2024-08-17T17:26:20.958240+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:26:20.958240+0000","last_undegraded":"2024-08-17T17:26:20.958240+0000","last_fullsized":"2024-08-17T17:26:20.958240+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":192,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.957756+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:26:20.957756+0000","last_peered":"2024-08-17T17:26:20.957756+0000","last_clean":"2024-08-17T17:26:20.957756+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:26:20.957756+0000","last_undegraded":"2024-08-17T17:26:20.957756+0000","last_fullsized":"2024-08-17T17:26:20.957756+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"172'1568","reported_seq":1589,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:23.940438+0000","last_change":"2024-08-17T17:26:02.536369+0000","last_active":"2024-08-17T17:26:23.940438+0000","last_peered":"2024-08-17T17:26:23.940438+0000","last_clean":"2024-08-17T17:26:23.940438+0000","last_became_active":"2024-08-17T17:26:02.536022+0000","last_became_peered":"2024-08-17T17:26:02.536022+0000","last_unstale":"2024-08-17T17:26:23.940438+0000","last_undegraded":"2024-08-17T17:26:23.940438+0000","last_fullsized":"2024-08-17T17:26:23.940438+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1568,"log_dups_size":0,"ondisk_log_size":1568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:22.755373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102760448,"num_objects":1568,"num_object_clones":0,"num_object_copies":4704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1568,"num_write_kb":100352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.958148+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:26:20.958148+0000","last_peered":"2024-08-17T17:26:20.958148+0000","last_clean":"2024-08-17T17:26:20.958148+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:26:20.958148+0000","last_undegraded":"2024-08-17T17:26:20.958148+0000","last_fullsized":"2024-08-17T17:26:20.958148+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":735,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950708+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:26:20.950708+0000","last_peered":"2024-08-17T17:26:20.950708+0000","last_clean":"2024-08-17T17:26:20.950708+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:26:20.950708+0000","last_undegraded":"2024-08-17T17:26:20.950708+0000","last_fullsized":"2024-08-17T17:26:20.950708+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947344+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:26:20.947344+0000","last_peered":"2024-08-17T17:26:20.947344+0000","last_clean":"2024-08-17T17:26:20.947344+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:26:20.947344+0000","last_undegraded":"2024-08-17T17:26:20.947344+0000","last_fullsized":"2024-08-17T17:26:20.947344+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"174'1841","reported_seq":1857,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-08-17T17:26:26.504949+0000","last_change":"2024-08-17T17:26:02.538931+0000","last_active":"2024-08-17T17:26:26.504949+0000","last_peered":"2024-08-17T17:26:26.504949+0000","last_clean":"2024-08-17T17:26:26.504949+0000","last_became_active":"2024-08-17T17:26:02.538728+0000","last_became_peered":"2024-08-17T17:26:02.538728+0000","last_unstale":"2024-08-17T17:26:26.504949+0000","last_undegraded":"2024-08-17T17:26:26.504949+0000","last_fullsized":"2024-08-17T17:26:26.504949+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1841,"log_dups_size":0,"ondisk_log_size":1841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:20.824450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120651776,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1841,"num_write_kb":117824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":198,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.964385+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:26:20.964385+0000","last_peered":"2024-08-17T17:26:20.964385+0000","last_clean":"2024-08-17T17:26:20.964385+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:26:20.964385+0000","last_undegraded":"2024-08-17T17:26:20.964385+0000","last_fullsized":"2024-08-17T17:26:20.964385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":705,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.958170+0000","last_change":"2024-08-17T17:25:52.491743+0000","last_active":"2024-08-17T17:26:20.958170+0000","last_peered":"2024-08-17T17:26:20.958170+0000","last_clean":"2024-08-17T17:26:20.958170+0000","last_became_active":"2024-08-17T17:25:52.491147+0000","last_became_peered":"2024-08-17T17:25:52.491147+0000","last_unstale":"2024-08-17T17:26:20.958170+0000","last_undegraded":"2024-08-17T17:26:20.958170+0000","last_fullsized":"2024-08-17T17:26:20.958170+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:20.776398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":195,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.962920+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:26:20.962920+0000","last_peered":"2024-08-17T17:26:20.962920+0000","last_clean":"2024-08-17T17:26:20.962920+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:26:20.962920+0000","last_undegraded":"2024-08-17T17:26:20.962920+0000","last_fullsized":"2024-08-17T17:26:20.962920+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"173'1746","reported_seq":1761,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.981699+0000","last_change":"2024-08-17T17:26:02.531249+0000","last_active":"2024-08-17T17:26:25.981699+0000","last_peered":"2024-08-17T17:26:25.981699+0000","last_clean":"2024-08-17T17:26:25.981699+0000","last_became_active":"2024-08-17T17:26:02.530950+0000","last_became_peered":"2024-08-17T17:26:02.530950+0000","last_unstale":"2024-08-17T17:26:25.981699+0000","last_undegraded":"2024-08-17T17:26:25.981699+0000","last_fullsized":"2024-08-17T17:26:25.981699+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":1746,"log_dups_size":0,"ondisk_log_size":1746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:20.431512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114425856,"num_objects":1746,"num_object_clones":0,"num_object_copies":5238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1746,"num_write_kb":111744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":217,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947429+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:26:20.947429+0000","last_peered":"2024-08-17T17:26:20.947429+0000","last_clean":"2024-08-17T17:26:20.947429+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:26:20.947429+0000","last_undegraded":"2024-08-17T17:26:20.947429+0000","last_fullsized":"2024-08-17T17:26:20.947429+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":417,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.952920+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:26:20.952920+0000","last_peered":"2024-08-17T17:26:20.952920+0000","last_clean":"2024-08-17T17:26:20.952920+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:26:20.952920+0000","last_undegraded":"2024-08-17T17:26:20.952920+0000","last_fullsized":"2024-08-17T17:26:20.952920+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":67,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983048+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:26:24.983048+0000","last_peered":"2024-08-17T17:26:24.983048+0000","last_clean":"2024-08-17T17:26:24.983048+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:26:24.983048+0000","last_undegraded":"2024-08-17T17:26:24.983048+0000","last_fullsized":"2024-08-17T17:26:24.983048+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983209+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:26:24.983209+0000","last_peered":"2024-08-17T17:26:24.983209+0000","last_clean":"2024-08-17T17:26:24.983209+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:26:24.983209+0000","last_undegraded":"2024-08-17T17:26:24.983209+0000","last_fullsized":"2024-08-17T17:26:24.983209+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":68,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983137+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:26:24.983137+0000","last_peered":"2024-08-17T17:26:24.983137+0000","last_clean":"2024-08-17T17:26:24.983137+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:26:24.983137+0000","last_undegraded":"2024-08-17T17:26:24.983137+0000","last_fullsized":"2024-08-17T17:26:24.983137+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":181,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.957607+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:26:20.957607+0000","last_peered":"2024-08-17T17:26:20.957607+0000","last_clean":"2024-08-17T17:26:20.957607+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:26:20.957607+0000","last_undegraded":"2024-08-17T17:26:20.957607+0000","last_fullsized":"2024-08-17T17:26:20.957607+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":168,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950735+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:26:20.950735+0000","last_peered":"2024-08-17T17:26:20.950735+0000","last_clean":"2024-08-17T17:26:20.950735+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:26:20.950735+0000","last_undegraded":"2024-08-17T17:26:20.950735+0000","last_fullsized":"2024-08-17T17:26:20.950735+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.950572+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:26:20.950572+0000","last_peered":"2024-08-17T17:26:20.950572+0000","last_clean":"2024-08-17T17:26:20.950572+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:26:20.950572+0000","last_undegraded":"2024-08-17T17:26:20.950572+0000","last_fullsized":"2024-08-17T17:26:20.950572+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":150,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.962594+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:26:20.962594+0000","last_peered":"2024-08-17T17:26:20.962594+0000","last_clean":"2024-08-17T17:26:20.962594+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:26:20.962594+0000","last_undegraded":"2024-08-17T17:26:20.962594+0000","last_fullsized":"2024-08-17T17:26:20.962594+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":197,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947660+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:26:20.947660+0000","last_peered":"2024-08-17T17:26:20.947660+0000","last_clean":"2024-08-17T17:26:20.947660+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:26:20.947660+0000","last_undegraded":"2024-08-17T17:26:20.947660+0000","last_fullsized":"2024-08-17T17:26:20.947660+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":199,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.962488+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:26:20.962488+0000","last_peered":"2024-08-17T17:26:20.962488+0000","last_clean":"2024-08-17T17:26:20.962488+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:26:20.962488+0000","last_undegraded":"2024-08-17T17:26:20.962488+0000","last_fullsized":"2024-08-17T17:26:20.962488+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.003618+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:26:25.003618+0000","last_peered":"2024-08-17T17:26:25.003618+0000","last_clean":"2024-08-17T17:26:25.003618+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:26:25.003618+0000","last_undegraded":"2024-08-17T17:26:25.003618+0000","last_fullsized":"2024-08-17T17:26:25.003618+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":215,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963234+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:26:20.963234+0000","last_peered":"2024-08-17T17:26:20.963234+0000","last_clean":"2024-08-17T17:26:20.963234+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:26:20.963234+0000","last_undegraded":"2024-08-17T17:26:20.963234+0000","last_fullsized":"2024-08-17T17:26:20.963234+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":191,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947578+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:26:20.947578+0000","last_peered":"2024-08-17T17:26:20.947578+0000","last_clean":"2024-08-17T17:26:20.947578+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:26:20.947578+0000","last_undegraded":"2024-08-17T17:26:20.947578+0000","last_fullsized":"2024-08-17T17:26:20.947578+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":195,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.958563+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:26:20.958563+0000","last_peered":"2024-08-17T17:26:20.958563+0000","last_clean":"2024-08-17T17:26:20.958563+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:26:20.958563+0000","last_undegraded":"2024-08-17T17:26:20.958563+0000","last_fullsized":"2024-08-17T17:26:20.958563+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":212,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963036+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:26:20.963036+0000","last_peered":"2024-08-17T17:26:20.963036+0000","last_clean":"2024-08-17T17:26:20.963036+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:26:20.963036+0000","last_undegraded":"2024-08-17T17:26:20.963036+0000","last_fullsized":"2024-08-17T17:26:20.963036+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983137+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:26:24.983137+0000","last_peered":"2024-08-17T17:26:24.983137+0000","last_clean":"2024-08-17T17:26:24.983137+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:26:24.983137+0000","last_undegraded":"2024-08-17T17:26:24.983137+0000","last_fullsized":"2024-08-17T17:26:24.983137+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":220,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.953409+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:26:20.953409+0000","last_peered":"2024-08-17T17:26:20.953409+0000","last_clean":"2024-08-17T17:26:20.953409+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:26:20.953409+0000","last_undegraded":"2024-08-17T17:26:20.953409+0000","last_fullsized":"2024-08-17T17:26:20.953409+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":197,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947226+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:26:20.947226+0000","last_peered":"2024-08-17T17:26:20.947226+0000","last_clean":"2024-08-17T17:26:20.947226+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:26:20.947226+0000","last_undegraded":"2024-08-17T17:26:20.947226+0000","last_fullsized":"2024-08-17T17:26:20.947226+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983142+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:26:24.983142+0000","last_peered":"2024-08-17T17:26:24.983142+0000","last_clean":"2024-08-17T17:26:24.983142+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:26:24.983142+0000","last_undegraded":"2024-08-17T17:26:24.983142+0000","last_fullsized":"2024-08-17T17:26:24.983142+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":159,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983052+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:26:24.983052+0000","last_peered":"2024-08-17T17:26:24.983052+0000","last_clean":"2024-08-17T17:26:24.983052+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:26:24.983052+0000","last_undegraded":"2024-08-17T17:26:24.983052+0000","last_fullsized":"2024-08-17T17:26:24.983052+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983122+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:26:24.983122+0000","last_peered":"2024-08-17T17:26:24.983122+0000","last_clean":"2024-08-17T17:26:24.983122+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:26:24.983122+0000","last_undegraded":"2024-08-17T17:26:24.983122+0000","last_fullsized":"2024-08-17T17:26:24.983122+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":144,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.983249+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:26:24.983249+0000","last_peered":"2024-08-17T17:26:24.983249+0000","last_clean":"2024-08-17T17:26:24.983249+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:26:24.983249+0000","last_undegraded":"2024-08-17T17:26:24.983249+0000","last_fullsized":"2024-08-17T17:26:24.983249+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":217,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.963098+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:26:20.963098+0000","last_peered":"2024-08-17T17:26:20.963098+0000","last_clean":"2024-08-17T17:26:20.963098+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:26:20.963098+0000","last_undegraded":"2024-08-17T17:26:20.963098+0000","last_fullsized":"2024-08-17T17:26:20.963098+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":205,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.965703+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:26:20.965703+0000","last_peered":"2024-08-17T17:26:20.965703+0000","last_clean":"2024-08-17T17:26:20.965703+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:26:20.965703+0000","last_undegraded":"2024-08-17T17:26:20.965703+0000","last_fullsized":"2024-08-17T17:26:20.965703+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.982865+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:26:24.982865+0000","last_peered":"2024-08-17T17:26:24.982865+0000","last_clean":"2024-08-17T17:26:24.982865+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:26:24.982865+0000","last_undegraded":"2024-08-17T17:26:24.982865+0000","last_fullsized":"2024-08-17T17:26:24.982865+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":208,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-08-17T17:26:20.947553+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:26:20.947553+0000","last_peered":"2024-08-17T17:26:20.947553+0000","last_clean":"2024-08-17T17:26:20.947553+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:26:20.947553+0000","last_undegraded":"2024-08-17T17:26:20.947553+0000","last_fullsized":"2024-08-17T17:26:20.947553+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1725825024,"num_objects":26334,"num_object_clones":0,"num_object_copies":79002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26334,"num_write_kb":1685376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5130551296,"data_stored":5130551296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":26334,"ondisk_log_size":26334,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201555,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972228,"kb_used_data":806584,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":92772924,"statfs":{"total":95995035648,"available":94999474176,"internally_reserved":0,"allocated":825942016,"data_stored":824951976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":169605579},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463193,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":804604,"kb_used_data":657008,"kb_used_omap":1,"kb_used_meta":147582,"kb_avail":92940548,"statfs":{"total":95995035648,"available":95171121152,"internally_reserved":0,"allocated":672776192,"data_stored":671794534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":151124423},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986465,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836556,"kb_used_data":598716,"kb_used_omap":1,"kb_used_meta":237822,"kb_avail":92908596,"statfs":{"total":95995035648,"available":95138402304,"internally_reserved":0,"allocated":613085184,"data_stored":612098715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":243530182},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435356,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064544,"kb_used_data":938452,"kb_used_omap":5,"kb_used_meta":126074,"kb_avail":92680608,"statfs":{"total":95995035648,"available":94904942592,"internally_reserved":0,"allocated":960974848,"data_stored":960008021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":129099849},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598882,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117920,"kb_used_data":879380,"kb_used_omap":7,"kb_used_meta":238520,"kb_avail":92627232,"statfs":{"total":95995035648,"available":94850285568,"internally_reserved":0,"allocated":900485120,"data_stored":899518749,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":244244653},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762409,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":714964,"kb_used_data":566668,"kb_used_omap":7,"kb_used_meta":148280,"kb_avail":93030188,"statfs":{"total":95995035648,"available":95262912512,"internally_reserved":0,"allocated":580268032,"data_stored":579308983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":151839137},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925935,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":610856,"kb_used_data":384800,"kb_used_omap":7,"kb_used_meta":226040,"kb_avail":93134296,"statfs":{"total":95995035648,"available":95369519104,"internally_reserved":0,"allocated":394035200,"data_stored":393057365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":231465564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":396165120,"data_stored":396165120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":622657536,"data_stored":622657536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":950140928,"data_stored":950140928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1022361600,"data_stored":1022361600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":620625920,"data_stored":620625920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":695271424,"data_stored":695271424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":823328768,"data_stored":823328768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:26:29.343 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:26:29.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_1 pg_num 26 2024-08-17T17:26:29.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:29 smithi175 ceph-mon[101434]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:29.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:29.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:29.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:29.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:29.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:29.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:29.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:30.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 36249 36233 141.511 112.188 0.00871269 0.00706114 2024-08-17T17:26:30.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:30.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:30.140 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:30 smithi064 ceph-mon[104516]: pgmap v539: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 147 MiB/s wr, 2.34k op/s 2024-08-17T17:26:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:30 smithi064 ceph-mon[109246]: pgmap v539: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 147 MiB/s wr, 2.34k op/s 2024-08-17T17:26:30.242 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:30.266 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:30.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:30.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.400 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:30 smithi175 ceph-mon[101434]: pgmap v539: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 147 MiB/s wr, 2.34k op/s 2024-08-17T17:26:30.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:30.541 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:30.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:30.700 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:30.721 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:30.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:30.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:30.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:30.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 38024 38008 139.711 110.938 0.00769372 0.00715123 2024-08-17T17:26:31.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:31.085 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:31.187 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:31.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:31.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:31.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:31.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:31.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:31.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:31.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:31.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:31.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:31.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 39819 39803 138.182 112.188 0.0105474 0.00723157 2024-08-17T17:26:32.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:32 smithi064 ceph-mon[104516]: pgmap v540: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:26:32.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:32 smithi064 ceph-mon[109246]: pgmap v540: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:26:32.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:32.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:32.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:32 smithi175 ceph-mon[101434]: pgmap v540: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:26:32.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:32.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:32.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:32.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:32.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:32.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:32.853 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:32.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:32.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 41524 41508 136.517 106.562 0.0110271 0.00732013 2024-08-17T17:26:33.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:33.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:33.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:33.321 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:26:33.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:33.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:33.360 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:26:33.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:33.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:33.403 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:26:33.403 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:33.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:26:33.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:33.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:26:33.481 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:33.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:33.515 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:26:33.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:33.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:33.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:33.551 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:26:33.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:33.588 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:26:33.589 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:33.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.665 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:33.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:33.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:34.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:26:34.013558+0000 min lat: 0.00132797 max lat: 0.141443 avg lat: 0.0073988 2024-08-17T17:26:34.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:26:34.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 43223 43207 135 106.188 0.00222923 0.0073988 2024-08-17T17:26:34.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:34.048 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:34.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:34.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:34.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:34.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:34.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:34 smithi175 ceph-mon[101434]: pgmap v541: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:26:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2941420684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:34 smithi064 ceph-mon[104516]: pgmap v541: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:26:34.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2941420684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:34.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:33] "GET /metrics HTTP/1.1" 200 36639 "" "Prometheus/2.51.0" 2024-08-17T17:26:34.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:34 smithi064 ceph-mon[109246]: pgmap v541: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:26:34.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2941420684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:34.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:34.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:34.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:34.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:34.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:34.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:34.852 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:34.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:34.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:34.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:35.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 44746 44730 133.104 95.1875 0.0104818 0.00750751 2024-08-17T17:26:35.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:35.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:35.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:35.249 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:35.351 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:35.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:35.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:35.514 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:35.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:35.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:35.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:35.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:35.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:35.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:35.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:36.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 46392 46376 131.726 102.875 0.00794299 0.00758592 2024-08-17T17:26:36.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:36.148 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:36.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:36.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:36.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:36 smithi175 ceph-mon[101434]: pgmap v542: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:26:36.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:36.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:36 smithi064 ceph-mon[104516]: pgmap v542: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:26:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:36 smithi064 ceph-mon[109246]: pgmap v542: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:26:36.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:36.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:36.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:36.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:36.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:36.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:36.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:37.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:37.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 47988 47972 130.335 99.75 0.014828 0.00766676 2024-08-17T17:26:37.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:37.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:37.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:37.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:37.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:37.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:37.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:37.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:37.548 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:37.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:37.677 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:37.779 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:37.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:37.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:37.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:38.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 49653 49637 129.24 104.062 0.00984518 0.00773176 2024-08-17T17:26:38.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:38.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:38.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:38.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:38.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:38.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:38.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:38 smithi064 ceph-mon[104516]: pgmap v543: 89 pgs: 89 active+clean; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:26:38.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:38 smithi064 ceph-mon[109246]: pgmap v543: 89 pgs: 89 active+clean; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:26:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:38 smithi175 ceph-mon[101434]: pgmap v543: 89 pgs: 89 active+clean; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:26:38.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:38.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:38.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.623 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:38.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:38.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:38.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:38.852 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:38.877 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:38.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:39.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 51016 51000 127.478 85.1875 0.0111318 0.00783698 2024-08-17T17:26:39.020 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:39.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:39.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:39.263 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:39.287 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:39.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:39.341 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:39.341 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:39.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:39.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:39.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:39.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:39.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:39.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.692 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:39.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:39.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:39.793 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:39.816 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:39.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:39.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:40.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 52138 52122 125.272 70.125 0.00658782 0.00797668 2024-08-17T17:26:40.051 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:26:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pg_num to 26 2024-08-17T17:26:40.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:40.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:40.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:40.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:40 smithi175 ceph-mon[101434]: pgmap v544: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:26:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1561040427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:40 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[104516]: pgmap v544: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1561040427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[109246]: pgmap v544: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1561040427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:26:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:40.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:40.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.516 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:40.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:26:40.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:40.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:40.785 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:40.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:40.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:41.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 53338 53322 123.41 75 0.00913992 0.00809737 2024-08-17T17:26:41.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:41.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:41.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:41.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:26:41.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:41.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[104516]: osdmap e175: 8 total, 8 up, 7 in 2024-08-17T17:26:41.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:26:41.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:41.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:26:41.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:41.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[109246]: osdmap e175: 8 total, 8 up, 7 in 2024-08-17T17:26:41.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:26:41.209 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:41.234 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:41.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:41.357 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:41 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:41 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:26:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1552818160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:41 smithi175 ceph-mon[101434]: osdmap e175: 8 total, 8 up, 7 in 2024-08-17T17:26:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:26:41.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:41.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:41.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:41.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:41.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:41.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:41.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:41.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:41.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:41.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:41.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:42.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 53485 53469 119.331 9.1875 0.10016 0.00810954 2024-08-17T17:26:42.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:42.103 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:42.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:42.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:42.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[104516]: pgmap v546: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-17T17:26:42.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[104516]: osdmap e176: 8 total, 8 up, 7 in 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[109246]: pgmap v546: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[109246]: osdmap e176: 8 total, 8 up, 7 in 2024-08-17T17:26:42.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:42.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:42.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:42.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:42.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:42 smithi175 ceph-mon[101434]: pgmap v546: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-17T17:26:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:26:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:42 smithi175 ceph-mon[101434]: osdmap e176: 8 total, 8 up, 7 in 2024-08-17T17:26:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:42.411 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:42.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:42.477 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:42.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:42.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:42.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:42.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:42.817 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:42.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:42.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:42.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:43.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 54968 54952 118.409 92.6875 0.00605648 0.00843951 2024-08-17T17:26:43.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:43.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:43.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:43.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:43.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:43.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:43 smithi175 ceph-mon[101434]: osdmap e177: 8 total, 8 up, 7 in 2024-08-17T17:26:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:43 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:26:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:43 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:26:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[104516]: osdmap e177: 8 total, 8 up, 7 in 2024-08-17T17:26:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:26:43.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:26:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[109246]: osdmap e177: 8 total, 8 up, 7 in 2024-08-17T17:26:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:26:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:26:43.497 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:43.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:43.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:43.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:43.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:43.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:43.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:43.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:43.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:43.925 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:43.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:43.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:26:44.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 56687 56671 118.04 107.438 0.00794488 0.00846525 2024-08-17T17:26:44.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:44.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:44.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:44.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:44.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.349 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:44.381 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:44 smithi175 ceph-mon[101434]: pgmap v549: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:26:44.382 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:44 smithi175 ceph-mon[101434]: 6.4 scrub starts 2024-08-17T17:26:44.382 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:44 smithi175 ceph-mon[101434]: 6.16 scrub starts 2024-08-17T17:26:44.382 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:44 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:26:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:43] "GET /metrics HTTP/1.1" 200 36640 "" "Prometheus/2.51.0" 2024-08-17T17:26:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[104516]: pgmap v549: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:26:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[104516]: 6.4 scrub starts 2024-08-17T17:26:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[104516]: 6.16 scrub starts 2024-08-17T17:26:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:26:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[109246]: pgmap v549: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:26:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[109246]: 6.4 scrub starts 2024-08-17T17:26:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[109246]: 6.16 scrub starts 2024-08-17T17:26:44.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:44 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:26:44.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:44.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:44.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:44.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:44.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:44.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:44.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:44.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:45.008 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:45.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 58276 58260 117.435 99.3125 0.0086898 0.00851013 2024-08-17T17:26:45.035 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:45.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:45.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.167 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:45.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:45.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:45.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:45.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:45 smithi175 ceph-mon[101434]: 6.1 scrub starts 2024-08-17T17:26:45.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:45 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:26:45.409 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:45 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:26:45.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[104516]: 6.1 scrub starts 2024-08-17T17:26:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:26:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:26:45.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[109246]: 6.1 scrub starts 2024-08-17T17:26:45.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:26:45.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:45 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:26:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:45.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:45.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:45.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:45.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:45.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:45.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:45.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:45.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:45.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:45.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:46.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 59834 59818 116.808 97.375 0.00642561 0.00855558 2024-08-17T17:26:46.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:46.118 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:46.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:46.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:46.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:46.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:46 smithi175 ceph-mon[101434]: pgmap v550: 99 pgs: 1 peering, 4 unknown, 94 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:26:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:46 smithi175 ceph-mon[101434]: 6.11 scrub starts 2024-08-17T17:26:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:46 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:26:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:46 smithi175 ceph-mon[101434]: 6.10 scrub starts 2024-08-17T17:26:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[104516]: pgmap v550: 99 pgs: 1 peering, 4 unknown, 94 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[104516]: 6.11 scrub starts 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[104516]: 6.10 scrub starts 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[109246]: pgmap v550: 99 pgs: 1 peering, 4 unknown, 94 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[109246]: 6.11 scrub starts 2024-08-17T17:26:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:26:46.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:46 smithi064 ceph-mon[109246]: 6.10 scrub starts 2024-08-17T17:26:46.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:46.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:46.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:46.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:46.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:46.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:46.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:46.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:47.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 61596 61580 116.606 110.125 0.00683928 0.00857097 2024-08-17T17:26:47.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:47.079 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:47.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:47 smithi175 ceph-mon[101434]: 6.14 scrub starts 2024-08-17T17:26:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:26:47.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:47.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:47.314 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:47.339 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:47.376 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:26:47.376 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:26:47.376 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2024-08-17T17:26:47.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:47.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:47 smithi064 ceph-mon[104516]: 6.14 scrub starts 2024-08-17T17:26:47.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:26:47.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:47 smithi064 ceph-mon[109246]: 6.14 scrub starts 2024-08-17T17:26:47.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:26:47.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:47.464 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:47.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:47.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:47.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:47.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:47.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.955 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:47.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 62115 62099 114.13 32.4375 0.00195556 0.00856754 2024-08-17T17:26:48.057 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:48.078 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:48.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:48.200 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:48.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:48.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: pgmap v551: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: osdmap e178: 8 total, 8 up, 7 in 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:26:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:48 smithi175 ceph-mon[101434]: osdmap e179: 8 total, 8 up, 7 in 2024-08-17T17:26:48.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: pgmap v551: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:48.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: osdmap e178: 8 total, 8 up, 7 in 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[104516]: osdmap e179: 8 total, 8 up, 7 in 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: pgmap v551: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: osdmap e178: 8 total, 8 up, 7 in 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:48.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:26:48.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:26:48.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:48 smithi064 ceph-mon[109246]: osdmap e179: 8 total, 8 up, 7 in 2024-08-17T17:26:48.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:48.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:48.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:48.772 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:26:48.772 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:26:48.774 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:48.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:48.824 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:48.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:48.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:48.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:48.988 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:49.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 62115 62099 110.87 0 - 0.00856754 2024-08-17T17:26:49.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:49.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:49.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:49.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:49.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:49.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:49.342 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:26:49.342 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:26:49.342 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2024-08-17T17:26:49.342 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:26:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:49 smithi175 ceph-mon[101434]: 6.11 scrub starts 2024-08-17T17:26:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:49 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:26:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:49 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:26:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1897202477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:49 smithi175 ceph-mon[101434]: osdmap e180: 8 total, 8 up, 7 in 2024-08-17T17:26:49.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[104516]: 6.11 scrub starts 2024-08-17T17:26:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1897202477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[104516]: osdmap e180: 8 total, 8 up, 7 in 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[109246]: 6.11 scrub starts 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1897202477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:49 smithi064 ceph-mon[109246]: osdmap e180: 8 total, 8 up, 7 in 2024-08-17T17:26:49.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:49.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:49.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:49.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:49.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:49.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:49.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:49.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:49.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:50.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 62739 62723 108.873 19.5 0.00624398 0.00917972 2024-08-17T17:26:50.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:50.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:50.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:50.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:50.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:50.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:50.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:50.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:50.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:50 smithi175 ceph-mon[101434]: pgmap v554: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:50 smithi175 ceph-mon[101434]: 6.1 scrub starts 2024-08-17T17:26:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:50 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:26:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:50 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:26:50.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[104516]: pgmap v554: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:50.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[104516]: 6.1 scrub starts 2024-08-17T17:26:50.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:26:50.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:26:50.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[109246]: pgmap v554: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 9.4 KiB/s, 0 objects/s recovering 2024-08-17T17:26:50.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[109246]: 6.1 scrub starts 2024-08-17T17:26:50.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:26:50.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:50 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:26:50.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:50.519 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:50.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:50.652 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:50.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:50.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:50.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:50.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:50.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:51.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 64265 64249 108.508 95.375 0.0237505 0.00920915 2024-08-17T17:26:51.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:51.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:51.186 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:51.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:51.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:51.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:51 smithi064 ceph-mon[104516]: 6.16 scrub starts 2024-08-17T17:26:51.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:51 smithi064 ceph-mon[109246]: 6.16 scrub starts 2024-08-17T17:26:51.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:51.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:51.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:51.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:51.637 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:51.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:51 smithi175 ceph-mon[101434]: 6.16 scrub starts 2024-08-17T17:26:51.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:51.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:51.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:52.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 65789 65773 108.159 95.25 0.0112691 0.00924061 2024-08-17T17:26:52.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:52.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:52.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:52.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:52.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:52.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:52.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:52.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:52.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:52 smithi064 ceph-mon[104516]: pgmap v556: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 11 KiB/s, 0 objects/s recovering 2024-08-17T17:26:52.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:52 smithi064 ceph-mon[109246]: pgmap v556: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 11 KiB/s, 0 objects/s recovering 2024-08-17T17:26:52.537 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:52.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:26:52.562 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:52 smithi175 ceph-mon[101434]: pgmap v556: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 11 KiB/s, 0 objects/s recovering 2024-08-17T17:26:52.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:52.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:52.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:52.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:52.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 67125 67109 107.527 83.5 0.00591136 0.00929147 2024-08-17T17:26:53.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:53.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:53.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:53.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:53.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:53.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:53 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:26:53.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:53 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:26:53.489 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:53.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:53.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:53.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:53.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:53 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:26:53.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:53.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:53.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:53.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:54.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:54.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:26:54.017613+0000 min lat: 0.00132797 max lat: 2.46805 avg lat: 0.00932295 2024-08-17T17:26:54.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:26:54.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 68581 68565 107.113 91 0.00152434 0.00932295 2024-08-17T17:26:54.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:54.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:54.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:54.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:26:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:26:53] "GET /metrics HTTP/1.1" 200 36640 "" "Prometheus/2.51.0" 2024-08-17T17:26:54.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.305 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:54.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:54.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:54.431 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:54.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:54.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[104516]: pgmap v557: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-08-17T17:26:54.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:26:54.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:26:54.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[109246]: pgmap v557: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-08-17T17:26:54.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:26:54.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:54 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:26:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.604 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:54.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:54 smithi175 ceph-mon[101434]: pgmap v557: 99 pgs: 1 remapped+peering, 2 active+clean+scrubbing, 96 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s 2024-08-17T17:26:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:54 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:26:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:54 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:26:54.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:54.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:54.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:54.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.910 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:54.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:55.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 69905 69889 106.519 82.75 0.00835241 0.00937988 2024-08-17T17:26:55.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:55.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.209 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:26:55.212 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:55.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:55.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/761472221' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:26:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/761472221' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:55.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:55.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:55.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.640 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:26:55.642 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/761472221' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:55.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:55.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:55.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:55.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:55.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:55.963 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:55.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:56.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 70682 70666 105.139 48.5625 0.00186751 0.00939853 2024-08-17T17:26:56.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:56.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:56.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:56.232 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:56.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:56.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.354 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:56.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:56.561 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: pgmap v558: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3550/200454 objects misplaced (1.771%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:56.561 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.561 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:26:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: osdmap e181: 8 total, 8 up, 7 in 2024-08-17T17:26:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:26:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:26:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:56 smithi175 ceph-mon[101434]: osdmap e182: 8 total, 8 up, 7 in 2024-08-17T17:26:56.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:56.604 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: pgmap v558: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3550/200454 objects misplaced (1.771%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: osdmap e181: 8 total, 8 up, 7 in 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[109246]: osdmap e182: 8 total, 8 up, 7 in 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: pgmap v558: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3550/200454 objects misplaced (1.771%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: osdmap e181: 8 total, 8 up, 7 in 2024-08-17T17:26:56.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:26:56.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:26:56.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:56 smithi064 ceph-mon[104516]: osdmap e182: 8 total, 8 up, 7 in 2024-08-17T17:26:56.706 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:56.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:56.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:56.853 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:56.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:56.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:57.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 70682 70666 102.694 0 - 0.00939853 2024-08-17T17:26:57.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:57.101 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:57.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:57.239 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:57.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:57.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:57.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:57.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:57.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:57.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.653 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:57.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:26:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:26:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:26:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:26:57.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:57 smithi175 ceph-mon[101434]: osdmap e183: 8 total, 8 up, 7 in 2024-08-17T17:26:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:26:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[104516]: osdmap e183: 8 total, 8 up, 7 in 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:26:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:57 smithi064 ceph-mon[109246]: osdmap e183: 8 total, 8 up, 7 in 2024-08-17T17:26:57.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:57.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:57.878 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:57.900 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:58.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:58.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 71221 71205 101.125 16.8438 0.00236411 0.00979638 2024-08-17T17:26:58.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:58.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:58.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:58.259 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[109246]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 3550/207288 objects misplaced (1.713%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:58.259 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[109246]: osdmap e184: 8 total, 8 up, 7 in 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[104516]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 3550/207288 objects misplaced (1.713%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:26:58.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:58 smithi064 ceph-mon[104516]: osdmap e184: 8 total, 8 up, 7 in 2024-08-17T17:26:58.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:58.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:58.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:58 smithi175 ceph-mon[101434]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 3550/207288 objects misplaced (1.713%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:26:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:58 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:26:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:58 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:26:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:58 smithi175 ceph-mon[101434]: osdmap e184: 8 total, 8 up, 7 in 2024-08-17T17:26:58.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:58.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:58.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:58.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:58.666 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:58.768 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:58.790 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:58.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:58.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:59.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 71283 71267 98.9639 3.875 0.00335354 0.00979066 2024-08-17T17:26:59.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:59.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:59.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:59.145 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:59.167 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:59.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:59.295 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:59.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:59.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:59.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.543 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:59.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[104516]: osdmap e185: 8 total, 8 up, 7 in 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:26:59.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:26:59 smithi064 ceph-mon[109246]: osdmap e185: 8 total, 8 up, 7 in 2024-08-17T17:26:59.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:59 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:26:59.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:59 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:26:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:26:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:26:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:26:59 smithi175 ceph-mon[101434]: osdmap e185: 8 total, 8 up, 7 in 2024-08-17T17:26:59.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:59.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:59.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:26:59.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:59.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:59.919 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:26:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:00.021 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:00.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 71683 71667 97.3351 25 0.00334853 0.0101313 2024-08-17T17:27:00.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:27:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:00.166 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:00.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:00.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:00.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:00.412 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.421 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:00.453 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":559,"stamp":"2024-08-17T17:26:55.216125+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4381939685,"num_objects":66818,"num_object_clones":0,"num_object_copies":200454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3550,"num_objects_unfound":0,"num_objects_dirty":66818,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":67913,"num_write_kb":4289601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":256,"num_bytes_recovered":20323511,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101363,"ondisk_log_size":101363,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14323812,"kb_used_data":12679272,"kb_used_omap":33,"kb_used_meta":1644382,"kb_avail":641892252,"statfs":{"total":671965249536,"available":657297666048,"internally_reserved":0,"allocated":12983574528,"data_stored":12976545192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":1683847755},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,3,5,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":570687488,"num_objects":8708,"num_object_clones":0,"num_object_copies":26124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3550,"num_objects_unfound":0,"num_objects_dirty":8708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8708,"num_write_kb":557312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.078798"},"pg_stats":[{"pgid":"6.16","version":"180'3880","reported_seq":3933,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.764501+0000","last_change":"2024-08-17T17:26:50.337409+0000","last_active":"2024-08-17T17:26:53.764501+0000","last_peered":"2024-08-17T17:26:53.764501+0000","last_clean":"2024-08-17T17:26:53.764501+0000","last_became_active":"2024-08-17T17:26:42.142775+0000","last_became_peered":"2024-08-17T17:26:42.142775+0000","last_unstale":"2024-08-17T17:26:53.764501+0000","last_undegraded":"2024-08-17T17:26:53.764501+0000","last_fullsized":"2024-08-17T17:26:53.764501+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3880,"log_dups_size":0,"ondisk_log_size":3880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:55.294707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143130624,"num_objects":2184,"num_object_clones":0,"num_object_copies":6552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2184,"num_write_kb":139776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":215,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190852+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:26:49.190852+0000","last_peered":"2024-08-17T17:26:49.190852+0000","last_clean":"2024-08-17T17:26:49.190852+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:26:49.190852+0000","last_undegraded":"2024-08-17T17:26:49.190852+0000","last_fullsized":"2024-08-17T17:26:49.190852+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":175,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249625+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:26:48.249625+0000","last_peered":"2024-08-17T17:26:48.249625+0000","last_clean":"2024-08-17T17:26:48.249625+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:26:48.249625+0000","last_undegraded":"2024-08-17T17:26:48.249625+0000","last_fullsized":"2024-08-17T17:26:48.249625+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"180'3790","reported_seq":3843,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.709259+0000","last_change":"2024-08-17T17:26:49.640452+0000","last_active":"2024-08-17T17:26:53.709259+0000","last_peered":"2024-08-17T17:26:53.709259+0000","last_clean":"2024-08-17T17:26:53.709259+0000","last_became_active":"2024-08-17T17:26:42.150796+0000","last_became_peered":"2024-08-17T17:26:42.150796+0000","last_unstale":"2024-08-17T17:26:53.709259+0000","last_undegraded":"2024-08-17T17:26:53.709259+0000","last_fullsized":"2024-08-17T17:26:53.709259+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3790,"log_dups_size":0,"ondisk_log_size":3790,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:54.276639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137527296,"num_objects":2099,"num_object_clones":0,"num_object_copies":6297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2099,"num_write_kb":134304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249594+0000","last_change":"2024-08-17T17:22:35.147844+0000","last_active":"2024-08-17T17:26:48.249594+0000","last_peered":"2024-08-17T17:26:48.249594+0000","last_clean":"2024-08-17T17:26:48.249594+0000","last_became_active":"2024-08-17T17:22:35.147446+0000","last_became_peered":"2024-08-17T17:22:35.147446+0000","last_unstale":"2024-08-17T17:26:48.249594+0000","last_undegraded":"2024-08-17T17:26:48.249594+0000","last_fullsized":"2024-08-17T17:26:48.249594+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:51.863397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.136944+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:26:49.136944+0000","last_peered":"2024-08-17T17:26:49.136944+0000","last_clean":"2024-08-17T17:26:49.136944+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:26:49.136944+0000","last_undegraded":"2024-08-17T17:26:49.136944+0000","last_fullsized":"2024-08-17T17:26:49.136944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"180'3898","reported_seq":3949,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.951040+0000","last_change":"2024-08-17T17:26:51.174561+0000","last_active":"2024-08-17T17:26:53.951040+0000","last_peered":"2024-08-17T17:26:53.951040+0000","last_clean":"2024-08-17T17:26:53.951040+0000","last_became_active":"2024-08-17T17:26:42.150530+0000","last_became_peered":"2024-08-17T17:26:42.150530+0000","last_unstale":"2024-08-17T17:26:53.951040+0000","last_undegraded":"2024-08-17T17:26:53.951040+0000","last_fullsized":"2024-08-17T17:26:53.951040+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3898,"log_dups_size":0,"ondisk_log_size":3898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:56.170126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143982592,"num_objects":2197,"num_object_clones":0,"num_object_copies":6591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2197,"num_write_kb":140608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":206,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137324+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:26:49.137324+0000","last_peered":"2024-08-17T17:26:49.137324+0000","last_clean":"2024-08-17T17:26:49.137324+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:26:49.137324+0000","last_undegraded":"2024-08-17T17:26:49.137324+0000","last_fullsized":"2024-08-17T17:26:49.137324+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249503+0000","last_change":"2024-08-17T17:22:21.218664+0000","last_active":"2024-08-17T17:26:48.249503+0000","last_peered":"2024-08-17T17:26:48.249503+0000","last_clean":"2024-08-17T17:26:48.249503+0000","last_became_active":"2024-08-17T17:22:21.218316+0000","last_became_peered":"2024-08-17T17:22:21.218316+0000","last_unstale":"2024-08-17T17:26:48.249503+0000","last_undegraded":"2024-08-17T17:26:48.249503+0000","last_fullsized":"2024-08-17T17:26:48.249503+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:04.394723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"180'3696","reported_seq":3728,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.981318+0000","last_change":"2024-08-17T17:26:42.160871+0000","last_active":"2024-08-17T17:26:50.981318+0000","last_peered":"2024-08-17T17:26:50.981318+0000","last_clean":"2024-08-17T17:26:50.981318+0000","last_became_active":"2024-08-17T17:26:42.158543+0000","last_became_peered":"2024-08-17T17:26:42.158543+0000","last_unstale":"2024-08-17T17:26:50.981318+0000","last_undegraded":"2024-08-17T17:26:50.981318+0000","last_fullsized":"2024-08-17T17:26:50.981318+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3696,"log_dups_size":0,"ondisk_log_size":3696,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131858432,"num_objects":2012,"num_object_clones":0,"num_object_copies":6036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2012,"num_write_kb":128768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":215,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191644+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:26:49.191644+0000","last_peered":"2024-08-17T17:26:49.191644+0000","last_clean":"2024-08-17T17:26:49.191644+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:26:49.191644+0000","last_undegraded":"2024-08-17T17:26:49.191644+0000","last_fullsized":"2024-08-17T17:26:49.191644+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":170,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142024+0000","last_change":"2024-08-17T17:22:35.164019+0000","last_active":"2024-08-17T17:26:48.142024+0000","last_peered":"2024-08-17T17:26:48.142024+0000","last_clean":"2024-08-17T17:26:48.142024+0000","last_became_active":"2024-08-17T17:22:35.145874+0000","last_became_peered":"2024-08-17T17:22:35.145874+0000","last_unstale":"2024-08-17T17:26:48.142024+0000","last_undegraded":"2024-08-17T17:26:48.142024+0000","last_fullsized":"2024-08-17T17:26:48.142024+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:17.044187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"180'4013","reported_seq":4033,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.595355+0000","last_change":"2024-08-17T17:26:02.534643+0000","last_active":"2024-08-17T17:26:50.595355+0000","last_peered":"2024-08-17T17:26:50.595355+0000","last_clean":"2024-08-17T17:26:50.595355+0000","last_became_active":"2024-08-17T17:26:02.534394+0000","last_became_peered":"2024-08-17T17:26:02.534394+0000","last_unstale":"2024-08-17T17:26:50.595355+0000","last_undegraded":"2024-08-17T17:26:50.595355+0000","last_fullsized":"2024-08-17T17:26:50.595355+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4013,"log_dups_size":0,"ondisk_log_size":4013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:29.181237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262995968,"num_objects":4013,"num_object_clones":0,"num_object_copies":12039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4013,"num_write_kb":256832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":170,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249851+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:26:48.249851+0000","last_peered":"2024-08-17T17:26:48.249851+0000","last_clean":"2024-08-17T17:26:48.249851+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:26:48.249851+0000","last_undegraded":"2024-08-17T17:26:48.249851+0000","last_fullsized":"2024-08-17T17:26:48.249851+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":58,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249327+0000","last_change":"2024-08-17T17:22:35.151064+0000","last_active":"2024-08-17T17:26:48.249327+0000","last_peered":"2024-08-17T17:26:48.249327+0000","last_clean":"2024-08-17T17:26:48.249327+0000","last_became_active":"2024-08-17T17:22:35.150715+0000","last_became_peered":"2024-08-17T17:22:35.150715+0000","last_unstale":"2024-08-17T17:26:48.249327+0000","last_undegraded":"2024-08-17T17:26:48.249327+0000","last_fullsized":"2024-08-17T17:26:48.249327+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:28.872846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"180'4171","reported_seq":4211,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.793685+0000","last_change":"2024-08-17T17:26:02.539050+0000","last_active":"2024-08-17T17:26:53.793685+0000","last_peered":"2024-08-17T17:26:53.793685+0000","last_clean":"2024-08-17T17:26:53.793685+0000","last_became_active":"2024-08-17T17:26:02.538574+0000","last_became_peered":"2024-08-17T17:26:02.538574+0000","last_unstale":"2024-08-17T17:26:53.793685+0000","last_undegraded":"2024-08-17T17:26:53.793685+0000","last_fullsized":"2024-08-17T17:26:53.793685+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4171,"log_dups_size":0,"ondisk_log_size":4171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:42.850082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273350656,"num_objects":4171,"num_object_clones":0,"num_object_copies":12513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4171,"num_write_kb":266944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":171,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249878+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:26:48.249878+0000","last_peered":"2024-08-17T17:26:48.249878+0000","last_clean":"2024-08-17T17:26:48.249878+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:26:48.249878+0000","last_undegraded":"2024-08-17T17:26:48.249878+0000","last_fullsized":"2024-08-17T17:26:48.249878+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":65,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247302+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:26:48.247302+0000","last_peered":"2024-08-17T17:26:48.247302+0000","last_clean":"2024-08-17T17:26:48.247302+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:26:48.247302+0000","last_undegraded":"2024-08-17T17:26:48.247302+0000","last_fullsized":"2024-08-17T17:26:48.247302+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"180'3683","reported_seq":3715,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.984279+0000","last_change":"2024-08-17T17:26:42.115209+0000","last_active":"2024-08-17T17:26:50.984279+0000","last_peered":"2024-08-17T17:26:50.984279+0000","last_clean":"2024-08-17T17:26:50.984279+0000","last_became_active":"2024-08-17T17:26:42.113429+0000","last_became_peered":"2024-08-17T17:26:42.113429+0000","last_unstale":"2024-08-17T17:26:50.984279+0000","last_undegraded":"2024-08-17T17:26:50.984279+0000","last_fullsized":"2024-08-17T17:26:50.984279+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3683,"log_dups_size":0,"ondisk_log_size":3683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131563520,"num_objects":2007,"num_object_clones":0,"num_object_copies":6021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2007,"num_write_kb":128480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":142,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249778+0000","last_change":"2024-08-17T17:22:35.152946+0000","last_active":"2024-08-17T17:26:48.249778+0000","last_peered":"2024-08-17T17:26:48.249778+0000","last_clean":"2024-08-17T17:26:48.249778+0000","last_became_active":"2024-08-17T17:22:35.146102+0000","last_became_peered":"2024-08-17T17:22:35.146102+0000","last_unstale":"2024-08-17T17:26:48.249778+0000","last_undegraded":"2024-08-17T17:26:48.249778+0000","last_fullsized":"2024-08-17T17:26:48.249778+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:26.168398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142096+0000","last_change":"2024-08-17T17:26:26.009991+0000","last_active":"2024-08-17T17:26:48.142096+0000","last_peered":"2024-08-17T17:26:48.142096+0000","last_clean":"2024-08-17T17:26:48.142096+0000","last_became_active":"2024-08-17T17:26:26.009678+0000","last_became_peered":"2024-08-17T17:26:26.009678+0000","last_unstale":"2024-08-17T17:26:48.142096+0000","last_undegraded":"2024-08-17T17:26:48.142096+0000","last_fullsized":"2024-08-17T17:26:48.142096+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:41.602469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"180'3776","reported_seq":3826,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.515123+0000","last_change":"2024-08-17T17:26:49.136803+0000","last_active":"2024-08-17T17:26:53.515123+0000","last_peered":"2024-08-17T17:26:53.515123+0000","last_clean":"2024-08-17T17:26:53.515123+0000","last_became_active":"2024-08-17T17:26:42.111639+0000","last_became_peered":"2024-08-17T17:26:42.111639+0000","last_unstale":"2024-08-17T17:26:53.515123+0000","last_undegraded":"2024-08-17T17:26:53.515123+0000","last_fullsized":"2024-08-17T17:26:53.515123+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3776,"log_dups_size":0,"ondisk_log_size":3776,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138739712,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2117,"num_write_kb":135488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191065+0000","last_change":"2024-08-17T17:22:35.152877+0000","last_active":"2024-08-17T17:26:49.191065+0000","last_peered":"2024-08-17T17:26:49.191065+0000","last_clean":"2024-08-17T17:26:49.191065+0000","last_became_active":"2024-08-17T17:22:35.152499+0000","last_became_peered":"2024-08-17T17:22:35.152499+0000","last_unstale":"2024-08-17T17:26:49.191065+0000","last_undegraded":"2024-08-17T17:26:49.191065+0000","last_fullsized":"2024-08-17T17:26:49.191065+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.210130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":221,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137225+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:26:49.137225+0000","last_peered":"2024-08-17T17:26:49.137225+0000","last_clean":"2024-08-17T17:26:49.137225+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:26:49.137225+0000","last_undegraded":"2024-08-17T17:26:49.137225+0000","last_fullsized":"2024-08-17T17:26:49.137225+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"180'4038","reported_seq":4060,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.520032+0000","last_change":"2024-08-17T17:26:02.533130+0000","last_active":"2024-08-17T17:26:51.520032+0000","last_peered":"2024-08-17T17:26:51.520032+0000","last_clean":"2024-08-17T17:26:51.520032+0000","last_became_active":"2024-08-17T17:26:02.532783+0000","last_became_peered":"2024-08-17T17:26:02.532783+0000","last_unstale":"2024-08-17T17:26:51.520032+0000","last_undegraded":"2024-08-17T17:26:51.520032+0000","last_fullsized":"2024-08-17T17:26:51.520032+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4038,"log_dups_size":0,"ondisk_log_size":4038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:32.674058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634368,"num_objects":4038,"num_object_clones":0,"num_object_copies":12114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4038,"num_write_kb":258432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":236,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191235+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:26:49.191235+0000","last_peered":"2024-08-17T17:26:49.191235+0000","last_clean":"2024-08-17T17:26:49.191235+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:26:49.191235+0000","last_undegraded":"2024-08-17T17:26:49.191235+0000","last_fullsized":"2024-08-17T17:26:49.191235+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.275980+0000","last_change":"2024-08-17T17:22:21.212458+0000","last_active":"2024-08-17T17:26:49.275980+0000","last_peered":"2024-08-17T17:26:49.275980+0000","last_clean":"2024-08-17T17:26:49.275980+0000","last_became_active":"2024-08-17T17:22:21.212121+0000","last_became_peered":"2024-08-17T17:22:21.212121+0000","last_unstale":"2024-08-17T17:26:49.275980+0000","last_undegraded":"2024-08-17T17:26:49.275980+0000","last_fullsized":"2024-08-17T17:26:49.275980+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:39.299181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"180'4271","reported_seq":4311,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.044037+0000","last_change":"2024-08-17T17:26:02.537749+0000","last_active":"2024-08-17T17:26:53.044037+0000","last_peered":"2024-08-17T17:26:53.044037+0000","last_clean":"2024-08-17T17:26:53.044037+0000","last_became_active":"2024-08-17T17:26:02.537124+0000","last_became_peered":"2024-08-17T17:26:02.537124+0000","last_unstale":"2024-08-17T17:26:53.044037+0000","last_undegraded":"2024-08-17T17:26:53.044037+0000","last_fullsized":"2024-08-17T17:26:53.044037+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4271,"log_dups_size":0,"ondisk_log_size":4271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.867512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279904256,"num_objects":4271,"num_object_clones":0,"num_object_copies":12813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4271,"num_write_kb":273344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142185+0000","last_change":"2024-08-17T17:22:21.204044+0000","last_active":"2024-08-17T17:26:48.142185+0000","last_peered":"2024-08-17T17:26:48.142185+0000","last_clean":"2024-08-17T17:26:48.142185+0000","last_became_active":"2024-08-17T17:22:21.203316+0000","last_became_peered":"2024-08-17T17:22:21.203316+0000","last_unstale":"2024-08-17T17:26:48.142185+0000","last_undegraded":"2024-08-17T17:26:48.142185+0000","last_fullsized":"2024-08-17T17:26:48.142185+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:38.185306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":65,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.276113+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:26:49.276113+0000","last_peered":"2024-08-17T17:26:49.276113+0000","last_clean":"2024-08-17T17:26:49.276113+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:26:49.276113+0000","last_undegraded":"2024-08-17T17:26:49.276113+0000","last_fullsized":"2024-08-17T17:26:49.276113+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"180'4316","reported_seq":4355,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.803893+0000","last_change":"2024-08-17T17:26:02.538371+0000","last_active":"2024-08-17T17:26:53.803893+0000","last_peered":"2024-08-17T17:26:53.803893+0000","last_clean":"2024-08-17T17:26:53.803893+0000","last_became_active":"2024-08-17T17:26:02.537919+0000","last_became_peered":"2024-08-17T17:26:02.537919+0000","last_unstale":"2024-08-17T17:26:53.803893+0000","last_undegraded":"2024-08-17T17:26:53.803893+0000","last_fullsized":"2024-08-17T17:26:53.803893+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4316,"log_dups_size":0,"ondisk_log_size":4316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:52.631309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":276224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":65,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247242+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:26:48.247242+0000","last_peered":"2024-08-17T17:26:48.247242+0000","last_clean":"2024-08-17T17:26:48.247242+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:26:48.247242+0000","last_undegraded":"2024-08-17T17:26:48.247242+0000","last_fullsized":"2024-08-17T17:26:48.247242+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":65,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.276203+0000","last_change":"2024-08-17T17:22:21.225592+0000","last_active":"2024-08-17T17:26:49.276203+0000","last_peered":"2024-08-17T17:26:49.276203+0000","last_clean":"2024-08-17T17:26:49.276203+0000","last_became_active":"2024-08-17T17:22:21.224770+0000","last_became_peered":"2024-08-17T17:22:21.224770+0000","last_unstale":"2024-08-17T17:26:49.276203+0000","last_undegraded":"2024-08-17T17:26:49.276203+0000","last_fullsized":"2024-08-17T17:26:49.276203+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:20.375077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"180'4251","reported_seq":4290,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.964849+0000","last_change":"2024-08-17T17:26:02.536813+0000","last_active":"2024-08-17T17:26:53.964849+0000","last_peered":"2024-08-17T17:26:53.964849+0000","last_clean":"2024-08-17T17:26:53.964849+0000","last_became_active":"2024-08-17T17:26:02.535602+0000","last_became_peered":"2024-08-17T17:26:02.535602+0000","last_unstale":"2024-08-17T17:26:53.964849+0000","last_undegraded":"2024-08-17T17:26:53.964849+0000","last_fullsized":"2024-08-17T17:26:53.964849+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":4251,"log_dups_size":0,"ondisk_log_size":4251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:05.335837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278593536,"num_objects":4251,"num_object_clones":0,"num_object_copies":12753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4251,"num_write_kb":272064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":216,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.193338+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:26:49.193338+0000","last_peered":"2024-08-17T17:26:49.193338+0000","last_clean":"2024-08-17T17:26:49.193338+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:26:49.193338+0000","last_undegraded":"2024-08-17T17:26:49.193338+0000","last_fullsized":"2024-08-17T17:26:49.193338+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":203,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191148+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:26:49.191148+0000","last_peered":"2024-08-17T17:26:49.191148+0000","last_clean":"2024-08-17T17:26:49.191148+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:26:49.191148+0000","last_undegraded":"2024-08-17T17:26:49.191148+0000","last_fullsized":"2024-08-17T17:26:49.191148+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"180'3846","reported_seq":3902,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.805344+0000","last_change":"2024-08-17T17:26:52.327771+0000","last_active":"2024-08-17T17:26:53.805344+0000","last_peered":"2024-08-17T17:26:53.805344+0000","last_clean":"2024-08-17T17:26:53.805344+0000","last_became_active":"2024-08-17T17:26:42.109279+0000","last_became_peered":"2024-08-17T17:26:42.109279+0000","last_unstale":"2024-08-17T17:26:53.805344+0000","last_undegraded":"2024-08-17T17:26:53.805344+0000","last_fullsized":"2024-08-17T17:26:53.805344+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3846,"log_dups_size":0,"ondisk_log_size":3846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:57.327418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141197312,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2154,"num_write_kb":137888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":475,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249664+0000","last_change":"2024-08-17T17:17:19.405259+0000","last_active":"2024-08-17T17:26:48.249664+0000","last_peered":"2024-08-17T17:26:48.249664+0000","last_clean":"2024-08-17T17:26:48.249664+0000","last_became_active":"2024-08-17T17:17:19.404913+0000","last_became_peered":"2024-08-17T17:17:19.404913+0000","last_unstale":"2024-08-17T17:26:48.249664+0000","last_undegraded":"2024-08-17T17:26:48.249664+0000","last_fullsized":"2024-08-17T17:26:48.249664+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:46.586318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":732,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190765+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:26:49.190765+0000","last_peered":"2024-08-17T17:26:49.190765+0000","last_clean":"2024-08-17T17:26:49.190765+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:26:49.190765+0000","last_undegraded":"2024-08-17T17:26:49.190765+0000","last_fullsized":"2024-08-17T17:26:49.190765+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":173,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.192256+0000","last_change":"2024-08-17T17:22:35.154205+0000","last_active":"2024-08-17T17:26:49.192256+0000","last_peered":"2024-08-17T17:26:49.192256+0000","last_clean":"2024-08-17T17:26:49.192256+0000","last_became_active":"2024-08-17T17:22:35.152868+0000","last_became_peered":"2024-08-17T17:22:35.152868+0000","last_unstale":"2024-08-17T17:26:49.192256+0000","last_undegraded":"2024-08-17T17:26:49.192256+0000","last_fullsized":"2024-08-17T17:26:49.192256+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:07.969524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142374+0000","last_change":"2024-08-17T17:22:21.207990+0000","last_active":"2024-08-17T17:26:48.142374+0000","last_peered":"2024-08-17T17:26:48.142374+0000","last_clean":"2024-08-17T17:26:48.142374+0000","last_became_active":"2024-08-17T17:22:21.207526+0000","last_became_peered":"2024-08-17T17:22:21.207526+0000","last_unstale":"2024-08-17T17:26:48.142374+0000","last_undegraded":"2024-08-17T17:26:48.142374+0000","last_fullsized":"2024-08-17T17:26:48.142374+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:04.778649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"180'3894","reported_seq":3944,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.960408+0000","last_change":"2024-08-17T17:26:50.157107+0000","last_active":"2024-08-17T17:26:53.960408+0000","last_peered":"2024-08-17T17:26:53.960408+0000","last_clean":"2024-08-17T17:26:53.960408+0000","last_became_active":"2024-08-17T17:26:42.115345+0000","last_became_peered":"2024-08-17T17:26:42.115345+0000","last_unstale":"2024-08-17T17:26:53.960408+0000","last_undegraded":"2024-08-17T17:26:53.960408+0000","last_fullsized":"2024-08-17T17:26:53.960408+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3894,"log_dups_size":0,"ondisk_log_size":3894,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:55.142745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143720448,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2193,"num_write_kb":140352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":553,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249447+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:26:48.249447+0000","last_peered":"2024-08-17T17:26:48.249447+0000","last_clean":"2024-08-17T17:26:48.249447+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:26:48.249447+0000","last_undegraded":"2024-08-17T17:26:48.249447+0000","last_fullsized":"2024-08-17T17:26:48.249447+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":221,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.136861+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:26:49.136861+0000","last_peered":"2024-08-17T17:26:49.136861+0000","last_clean":"2024-08-17T17:26:49.136861+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:26:49.136861+0000","last_undegraded":"2024-08-17T17:26:49.136861+0000","last_fullsized":"2024-08-17T17:26:49.136861+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":218,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137431+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:26:49.137431+0000","last_peered":"2024-08-17T17:26:49.137431+0000","last_clean":"2024-08-17T17:26:49.137431+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:26:49.137431+0000","last_undegraded":"2024-08-17T17:26:49.137431+0000","last_fullsized":"2024-08-17T17:26:49.137431+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"180'3738","reported_seq":3770,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.975161+0000","last_change":"2024-08-17T17:26:42.115007+0000","last_active":"2024-08-17T17:26:50.975161+0000","last_peered":"2024-08-17T17:26:50.975161+0000","last_clean":"2024-08-17T17:26:50.975161+0000","last_became_active":"2024-08-17T17:26:42.113234+0000","last_became_peered":"2024-08-17T17:26:42.113234+0000","last_unstale":"2024-08-17T17:26:50.975161+0000","last_undegraded":"2024-08-17T17:26:50.975161+0000","last_fullsized":"2024-08-17T17:26:50.975161+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3738,"log_dups_size":0,"ondisk_log_size":3738,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2054,"num_write_kb":131456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":618,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190679+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:26:49.190679+0000","last_peered":"2024-08-17T17:26:49.190679+0000","last_clean":"2024-08-17T17:26:49.190679+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:26:49.190679+0000","last_undegraded":"2024-08-17T17:26:49.190679+0000","last_fullsized":"2024-08-17T17:26:49.190679+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":198,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.308287+0000","last_change":"2024-08-17T17:18:22.906989+0000","last_active":"2024-08-17T17:26:49.308287+0000","last_peered":"2024-08-17T17:26:49.308287+0000","last_clean":"2024-08-17T17:26:49.308287+0000","last_became_active":"2024-08-17T17:18:22.906619+0000","last_became_peered":"2024-08-17T17:18:22.906619+0000","last_unstale":"2024-08-17T17:26:49.308287+0000","last_undegraded":"2024-08-17T17:26:49.308287+0000","last_fullsized":"2024-08-17T17:26:49.308287+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:16.359584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":239,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273579+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:26:49.273579+0000","last_peered":"2024-08-17T17:26:49.273579+0000","last_clean":"2024-08-17T17:26:49.273579+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:26:49.273579+0000","last_undegraded":"2024-08-17T17:26:49.273579+0000","last_fullsized":"2024-08-17T17:26:49.273579+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"180'3850","reported_seq":3906,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.715490+0000","last_change":"2024-08-17T17:26:53.302976+0000","last_active":"2024-08-17T17:26:53.715490+0000","last_peered":"2024-08-17T17:26:53.715490+0000","last_clean":"2024-08-17T17:26:53.715490+0000","last_became_active":"2024-08-17T17:26:42.136378+0000","last_became_peered":"2024-08-17T17:26:42.136378+0000","last_unstale":"2024-08-17T17:26:53.715490+0000","last_undegraded":"2024-08-17T17:26:53.715490+0000","last_fullsized":"2024-08-17T17:26:53.715490+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3850,"log_dups_size":0,"ondisk_log_size":3850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:58.301760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141164544,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2154,"num_write_kb":137856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1046,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273279+0000","last_change":"2024-08-17T17:25:52.490672+0000","last_active":"2024-08-17T17:26:49.273279+0000","last_peered":"2024-08-17T17:26:49.273279+0000","last_clean":"2024-08-17T17:26:49.273279+0000","last_became_active":"2024-08-17T17:25:52.490180+0000","last_became_peered":"2024-08-17T17:25:52.490180+0000","last_unstale":"2024-08-17T17:26:49.273279+0000","last_undegraded":"2024-08-17T17:26:49.273279+0000","last_fullsized":"2024-08-17T17:26:49.273279+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:46.014207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190528+0000","last_change":"2024-08-17T17:26:26.010304+0000","last_active":"2024-08-17T17:26:49.190528+0000","last_peered":"2024-08-17T17:26:49.190528+0000","last_clean":"2024-08-17T17:26:49.190528+0000","last_became_active":"2024-08-17T17:26:26.010001+0000","last_became_peered":"2024-08-17T17:26:26.010001+0000","last_unstale":"2024-08-17T17:26:49.190528+0000","last_undegraded":"2024-08-17T17:26:49.190528+0000","last_fullsized":"2024-08-17T17:26:49.190528+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:44.721039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":104,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273471+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:26:49.273471+0000","last_peered":"2024-08-17T17:26:49.273471+0000","last_clean":"2024-08-17T17:26:49.273471+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:26:49.273471+0000","last_undegraded":"2024-08-17T17:26:49.273471+0000","last_fullsized":"2024-08-17T17:26:49.273471+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"180'3769","reported_seq":3822,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.531694+0000","last_change":"2024-08-17T17:26:49.674340+0000","last_active":"2024-08-17T17:26:53.531694+0000","last_peered":"2024-08-17T17:26:53.531694+0000","last_clean":"2024-08-17T17:26:53.531694+0000","last_became_active":"2024-08-17T17:26:42.111775+0000","last_became_peered":"2024-08-17T17:26:42.111775+0000","last_unstale":"2024-08-17T17:26:53.531694+0000","last_undegraded":"2024-08-17T17:26:53.531694+0000","last_fullsized":"2024-08-17T17:26:53.531694+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3769,"log_dups_size":0,"ondisk_log_size":3769,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:54.368531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137953280,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2105,"num_write_kb":134720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191357+0000","last_change":"2024-08-17T17:22:35.145827+0000","last_active":"2024-08-17T17:26:49.191357+0000","last_peered":"2024-08-17T17:26:49.191357+0000","last_clean":"2024-08-17T17:26:49.191357+0000","last_became_active":"2024-08-17T17:22:35.145249+0000","last_became_peered":"2024-08-17T17:22:35.145249+0000","last_unstale":"2024-08-17T17:26:49.191357+0000","last_undegraded":"2024-08-17T17:26:49.191357+0000","last_fullsized":"2024-08-17T17:26:49.191357+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:40.129604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":502,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273570+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:26:49.273570+0000","last_peered":"2024-08-17T17:26:49.273570+0000","last_clean":"2024-08-17T17:26:49.273570+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:26:49.273570+0000","last_undegraded":"2024-08-17T17:26:49.273570+0000","last_fullsized":"2024-08-17T17:26:49.273570+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273475+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:26:49.273475+0000","last_peered":"2024-08-17T17:26:49.273475+0000","last_clean":"2024-08-17T17:26:49.273475+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:26:49.273475+0000","last_undegraded":"2024-08-17T17:26:49.273475+0000","last_fullsized":"2024-08-17T17:26:49.273475+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"180'3742","reported_seq":3792,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.950477+0000","last_change":"2024-08-17T17:26:49.273207+0000","last_active":"2024-08-17T17:26:53.950477+0000","last_peered":"2024-08-17T17:26:53.950477+0000","last_clean":"2024-08-17T17:26:53.950477+0000","last_became_active":"2024-08-17T17:26:42.117607+0000","last_became_peered":"2024-08-17T17:26:42.117607+0000","last_unstale":"2024-08-17T17:26:53.950477+0000","last_undegraded":"2024-08-17T17:26:53.950477+0000","last_fullsized":"2024-08-17T17:26:53.950477+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3742,"log_dups_size":0,"ondisk_log_size":3742,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:54.110526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138575872,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2114,"num_write_kb":135328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":205,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273509+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:26:49.273509+0000","last_peered":"2024-08-17T17:26:49.273509+0000","last_clean":"2024-08-17T17:26:49.273509+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:26:49.273509+0000","last_undegraded":"2024-08-17T17:26:49.273509+0000","last_fullsized":"2024-08-17T17:26:49.273509+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":742,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249914+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:26:48.249914+0000","last_peered":"2024-08-17T17:26:48.249914+0000","last_clean":"2024-08-17T17:26:48.249914+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:26:48.249914+0000","last_undegraded":"2024-08-17T17:26:48.249914+0000","last_fullsized":"2024-08-17T17:26:48.249914+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137203+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:26:49.137203+0000","last_peered":"2024-08-17T17:26:49.137203+0000","last_clean":"2024-08-17T17:26:49.137203+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:26:49.137203+0000","last_undegraded":"2024-08-17T17:26:49.137203+0000","last_fullsized":"2024-08-17T17:26:49.137203+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"180'3663","reported_seq":3695,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.487376+0000","last_change":"2024-08-17T17:26:42.116187+0000","last_active":"2024-08-17T17:26:51.487376+0000","last_peered":"2024-08-17T17:26:51.487376+0000","last_clean":"2024-08-17T17:26:51.487376+0000","last_became_active":"2024-08-17T17:26:42.111242+0000","last_became_peered":"2024-08-17T17:26:42.111242+0000","last_unstale":"2024-08-17T17:26:51.487376+0000","last_undegraded":"2024-08-17T17:26:51.487376+0000","last_fullsized":"2024-08-17T17:26:51.487376+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3663,"log_dups_size":0,"ondisk_log_size":3663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131596288,"num_objects":2008,"num_object_clones":0,"num_object_copies":6024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2008,"num_write_kb":128512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":214,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191847+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:26:49.191847+0000","last_peered":"2024-08-17T17:26:49.191847+0000","last_clean":"2024-08-17T17:26:49.191847+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:26:49.191847+0000","last_undegraded":"2024-08-17T17:26:49.191847+0000","last_fullsized":"2024-08-17T17:26:49.191847+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":722,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273974+0000","last_change":"2024-08-17T17:25:52.491743+0000","last_active":"2024-08-17T17:26:49.273974+0000","last_peered":"2024-08-17T17:26:49.273974+0000","last_clean":"2024-08-17T17:26:49.273974+0000","last_became_active":"2024-08-17T17:25:52.491147+0000","last_became_peered":"2024-08-17T17:25:52.491147+0000","last_unstale":"2024-08-17T17:26:49.273974+0000","last_undegraded":"2024-08-17T17:26:49.273974+0000","last_fullsized":"2024-08-17T17:26:49.273974+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:20.776398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":212,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.193286+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:26:49.193286+0000","last_peered":"2024-08-17T17:26:49.193286+0000","last_clean":"2024-08-17T17:26:49.193286+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:26:49.193286+0000","last_undegraded":"2024-08-17T17:26:49.193286+0000","last_fullsized":"2024-08-17T17:26:49.193286+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"180'3637","reported_seq":3667,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.991324+0000","last_change":"2024-08-17T17:26:42.123487+0000","last_active":"2024-08-17T17:26:50.991324+0000","last_peered":"2024-08-17T17:26:50.991324+0000","last_clean":"2024-08-17T17:26:50.991324+0000","last_became_active":"2024-08-17T17:26:42.121723+0000","last_became_peered":"2024-08-17T17:26:42.121723+0000","last_unstale":"2024-08-17T17:26:50.991324+0000","last_undegraded":"2024-08-17T17:26:50.991324+0000","last_fullsized":"2024-08-17T17:26:50.991324+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3637,"log_dups_size":0,"ondisk_log_size":3637,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1995,"num_write_kb":127680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":233,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137319+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:26:49.137319+0000","last_peered":"2024-08-17T17:26:49.137319+0000","last_clean":"2024-08-17T17:26:49.137319+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:26:49.137319+0000","last_undegraded":"2024-08-17T17:26:49.137319+0000","last_fullsized":"2024-08-17T17:26:49.137319+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":424,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142439+0000","last_change":"2024-08-17T17:18:22.902422+0000","last_active":"2024-08-17T17:26:48.142439+0000","last_peered":"2024-08-17T17:26:48.142439+0000","last_clean":"2024-08-17T17:26:48.142439+0000","last_became_active":"2024-08-17T17:18:22.901986+0000","last_became_peered":"2024-08-17T17:18:22.901986+0000","last_unstale":"2024-08-17T17:26:48.142439+0000","last_undegraded":"2024-08-17T17:26:48.142439+0000","last_fullsized":"2024-08-17T17:26:48.142439+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:18.530556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247376+0000","last_change":"2024-08-17T17:22:35.155953+0000","last_active":"2024-08-17T17:26:48.247376+0000","last_peered":"2024-08-17T17:26:48.247376+0000","last_clean":"2024-08-17T17:26:48.247376+0000","last_became_active":"2024-08-17T17:22:35.155499+0000","last_became_peered":"2024-08-17T17:22:35.155499+0000","last_unstale":"2024-08-17T17:26:48.247376+0000","last_undegraded":"2024-08-17T17:26:48.247376+0000","last_fullsized":"2024-08-17T17:26:48.247376+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:48.657567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.11","version":"180'3734","reported_seq":3785,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.950599+0000","last_change":"2024-08-17T17:26:53.201233+0000","last_active":"2024-08-17T17:26:53.950599+0000","last_peered":"2024-08-17T17:26:53.950599+0000","last_clean":"2024-08-17T17:26:53.950599+0000","last_became_active":"2024-08-17T17:26:42.130582+0000","last_became_peered":"2024-08-17T17:26:42.130582+0000","last_unstale":"2024-08-17T17:26:53.950599+0000","last_undegraded":"2024-08-17T17:26:53.950599+0000","last_fullsized":"2024-08-17T17:26:53.950599+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3734,"log_dups_size":0,"ondisk_log_size":3734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:58.178316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138051584,"num_objects":2107,"num_object_clones":0,"num_object_copies":6321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2107,"num_write_kb":134816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247566+0000","last_change":"2024-08-17T17:22:21.224370+0000","last_active":"2024-08-17T17:26:48.247566+0000","last_peered":"2024-08-17T17:26:48.247566+0000","last_clean":"2024-08-17T17:26:48.247566+0000","last_became_active":"2024-08-17T17:22:21.223847+0000","last_became_peered":"2024-08-17T17:22:21.223847+0000","last_unstale":"2024-08-17T17:26:48.247566+0000","last_undegraded":"2024-08-17T17:26:48.247566+0000","last_fullsized":"2024-08-17T17:26:48.247566+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:24.550534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":74,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247461+0000","last_change":"2024-08-17T17:22:21.231511+0000","last_active":"2024-08-17T17:26:48.247461+0000","last_peered":"2024-08-17T17:26:48.247461+0000","last_clean":"2024-08-17T17:26:48.247461+0000","last_became_active":"2024-08-17T17:22:21.210815+0000","last_became_peered":"2024-08-17T17:22:21.210815+0000","last_unstale":"2024-08-17T17:26:48.247461+0000","last_undegraded":"2024-08-17T17:26:48.247461+0000","last_fullsized":"2024-08-17T17:26:48.247461+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:02.343329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.10","version":"180'3779","reported_seq":4294,"reported_epoch":180,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:26:53.547694+0000","last_change":"2024-08-17T17:26:49.626117+0000","last_active":"2024-08-17T17:26:53.547694+0000","last_peered":"2024-08-17T17:26:53.547694+0000","last_clean":"2024-08-17T17:26:47.091931+0000","last_became_active":"2024-08-17T17:26:49.605767+0000","last_became_peered":"2024-08-17T17:26:49.605767+0000","last_unstale":"2024-08-17T17:26:53.547694+0000","last_undegraded":"2024-08-17T17:26:53.547694+0000","last_fullsized":"2024-08-17T17:26:49.136740+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3779,"log_dups_size":0,"ondisk_log_size":3779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3550,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2115,"num_write_kb":135360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1","objects":340},{"shards":"0,2,3","objects":1775}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":197,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273279+0000","last_change":"2024-08-17T17:18:22.909634+0000","last_active":"2024-08-17T17:26:49.273279+0000","last_peered":"2024-08-17T17:26:49.273279+0000","last_clean":"2024-08-17T17:26:49.273279+0000","last_became_active":"2024-08-17T17:18:22.909248+0000","last_became_peered":"2024-08-17T17:18:22.909248+0000","last_unstale":"2024-08-17T17:26:49.273279+0000","last_undegraded":"2024-08-17T17:26:49.273279+0000","last_fullsized":"2024-08-17T17:26:49.273279+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:39.314130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":175,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249647+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:26:48.249647+0000","last_peered":"2024-08-17T17:26:48.249647+0000","last_clean":"2024-08-17T17:26:48.249647+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:26:48.249647+0000","last_undegraded":"2024-08-17T17:26:48.249647+0000","last_fullsized":"2024-08-17T17:26:48.249647+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.13","version":"180'3604","reported_seq":3639,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.911840+0000","last_change":"2024-08-17T17:26:42.135873+0000","last_active":"2024-08-17T17:26:50.911840+0000","last_peered":"2024-08-17T17:26:50.911840+0000","last_clean":"2024-08-17T17:26:50.911840+0000","last_became_active":"2024-08-17T17:26:42.130082+0000","last_became_peered":"2024-08-17T17:26:42.130082+0000","last_unstale":"2024-08-17T17:26:50.911840+0000","last_undegraded":"2024-08-17T17:26:50.911840+0000","last_fullsized":"2024-08-17T17:26:50.911840+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3604,"log_dups_size":0,"ondisk_log_size":3604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128581632,"num_objects":1962,"num_object_clones":0,"num_object_copies":5886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1962,"num_write_kb":125568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":109,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.249785+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:26:48.249785+0000","last_peered":"2024-08-17T17:26:48.249785+0000","last_clean":"2024-08-17T17:26:48.249785+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:26:48.249785+0000","last_undegraded":"2024-08-17T17:26:48.249785+0000","last_fullsized":"2024-08-17T17:26:48.249785+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190545+0000","last_change":"2024-08-17T17:18:55.498356+0000","last_active":"2024-08-17T17:26:49.190545+0000","last_peered":"2024-08-17T17:26:49.190545+0000","last_clean":"2024-08-17T17:26:49.190545+0000","last_became_active":"2024-08-17T17:18:55.497704+0000","last_became_peered":"2024-08-17T17:18:55.497704+0000","last_unstale":"2024-08-17T17:26:49.190545+0000","last_undegraded":"2024-08-17T17:26:49.190545+0000","last_fullsized":"2024-08-17T17:26:49.190545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:01.077927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.12","version":"180'3683","reported_seq":3715,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.474940+0000","last_change":"2024-08-17T17:26:42.151894+0000","last_active":"2024-08-17T17:26:51.474940+0000","last_peered":"2024-08-17T17:26:51.474940+0000","last_clean":"2024-08-17T17:26:51.474940+0000","last_became_active":"2024-08-17T17:26:42.150462+0000","last_became_peered":"2024-08-17T17:26:42.150462+0000","last_unstale":"2024-08-17T17:26:51.474940+0000","last_undegraded":"2024-08-17T17:26:51.474940+0000","last_fullsized":"2024-08-17T17:26:51.474940+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3683,"log_dups_size":0,"ondisk_log_size":3683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132907008,"num_objects":2028,"num_object_clones":0,"num_object_copies":6084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2028,"num_write_kb":129792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":213,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137058+0000","last_change":"2024-08-17T17:22:35.146377+0000","last_active":"2024-08-17T17:26:49.137058+0000","last_peered":"2024-08-17T17:26:49.137058+0000","last_clean":"2024-08-17T17:26:49.137058+0000","last_became_active":"2024-08-17T17:22:35.146015+0000","last_became_peered":"2024-08-17T17:22:35.146015+0000","last_unstale":"2024-08-17T17:26:49.137058+0000","last_undegraded":"2024-08-17T17:26:49.137058+0000","last_fullsized":"2024-08-17T17:26:49.137058+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:47.265435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":215,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.193167+0000","last_change":"2024-08-17T17:18:55.497153+0000","last_active":"2024-08-17T17:26:49.193167+0000","last_peered":"2024-08-17T17:26:49.193167+0000","last_clean":"2024-08-17T17:26:49.193167+0000","last_became_active":"2024-08-17T17:18:55.496237+0000","last_became_peered":"2024-08-17T17:18:55.496237+0000","last_unstale":"2024-08-17T17:26:49.193167+0000","last_undegraded":"2024-08-17T17:26:49.193167+0000","last_fullsized":"2024-08-17T17:26:49.193167+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:04.272008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.142396+0000","last_change":"2024-08-17T17:22:35.161808+0000","last_active":"2024-08-17T17:26:48.142396+0000","last_peered":"2024-08-17T17:26:48.142396+0000","last_clean":"2024-08-17T17:26:48.142396+0000","last_became_active":"2024-08-17T17:22:35.161296+0000","last_became_peered":"2024-08-17T17:22:35.161296+0000","last_unstale":"2024-08-17T17:26:48.142396+0000","last_undegraded":"2024-08-17T17:26:48.142396+0000","last_fullsized":"2024-08-17T17:26:48.142396+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:29.151129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.193327+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:26:49.193327+0000","last_peered":"2024-08-17T17:26:49.193327+0000","last_clean":"2024-08-17T17:26:49.193327+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:26:49.193327+0000","last_undegraded":"2024-08-17T17:26:49.193327+0000","last_fullsized":"2024-08-17T17:26:49.193327+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137416+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:26:49.137416+0000","last_peered":"2024-08-17T17:26:49.137416+0000","last_clean":"2024-08-17T17:26:49.137416+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:26:49.137416+0000","last_undegraded":"2024-08-17T17:26:49.137416+0000","last_fullsized":"2024-08-17T17:26:49.137416+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":211,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.273321+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:26:49.273321+0000","last_peered":"2024-08-17T17:26:49.273321+0000","last_clean":"2024-08-17T17:26:49.273321+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:26:49.273321+0000","last_undegraded":"2024-08-17T17:26:49.273321+0000","last_fullsized":"2024-08-17T17:26:49.273321+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191109+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:26:49.191109+0000","last_peered":"2024-08-17T17:26:49.191109+0000","last_clean":"2024-08-17T17:26:49.191109+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:26:49.191109+0000","last_undegraded":"2024-08-17T17:26:49.191109+0000","last_fullsized":"2024-08-17T17:26:49.191109+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":64,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247547+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:26:48.247547+0000","last_peered":"2024-08-17T17:26:48.247547+0000","last_clean":"2024-08-17T17:26:48.247547+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:26:48.247547+0000","last_undegraded":"2024-08-17T17:26:48.247547+0000","last_fullsized":"2024-08-17T17:26:48.247547+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190522+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:26:49.190522+0000","last_peered":"2024-08-17T17:26:49.190522+0000","last_clean":"2024-08-17T17:26:49.190522+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:26:49.190522+0000","last_undegraded":"2024-08-17T17:26:49.190522+0000","last_fullsized":"2024-08-17T17:26:49.190522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":213,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137116+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:26:49.137116+0000","last_peered":"2024-08-17T17:26:49.137116+0000","last_clean":"2024-08-17T17:26:49.137116+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:26:49.137116+0000","last_undegraded":"2024-08-17T17:26:49.137116+0000","last_fullsized":"2024-08-17T17:26:49.137116+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"180'3760","reported_seq":3813,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:53.472977+0000","last_change":"2024-08-17T17:26:53.383726+0000","last_active":"2024-08-17T17:26:53.472977+0000","last_peered":"2024-08-17T17:26:53.472977+0000","last_clean":"2024-08-17T17:26:53.472977+0000","last_became_active":"2024-08-17T17:26:42.126197+0000","last_became_peered":"2024-08-17T17:26:42.126197+0000","last_unstale":"2024-08-17T17:26:53.472977+0000","last_undegraded":"2024-08-17T17:26:53.472977+0000","last_fullsized":"2024-08-17T17:26:53.472977+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3760,"log_dups_size":0,"ondisk_log_size":3760,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:26:58.367198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137691136,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2101,"num_write_kb":134464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247592+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:26:48.247592+0000","last_peered":"2024-08-17T17:26:48.247592+0000","last_clean":"2024-08-17T17:26:48.247592+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:26:48.247592+0000","last_undegraded":"2024-08-17T17:26:48.247592+0000","last_fullsized":"2024-08-17T17:26:48.247592+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247458+0000","last_change":"2024-08-17T17:22:35.166937+0000","last_active":"2024-08-17T17:26:48.247458+0000","last_peered":"2024-08-17T17:26:48.247458+0000","last_clean":"2024-08-17T17:26:48.247458+0000","last_became_active":"2024-08-17T17:22:35.161731+0000","last_became_peered":"2024-08-17T17:22:35.161731+0000","last_unstale":"2024-08-17T17:26:48.247458+0000","last_undegraded":"2024-08-17T17:26:48.247458+0000","last_fullsized":"2024-08-17T17:26:48.247458+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:52.247635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"180'3699","reported_seq":3731,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:50.984918+0000","last_change":"2024-08-17T17:26:42.161996+0000","last_active":"2024-08-17T17:26:50.984918+0000","last_peered":"2024-08-17T17:26:50.984918+0000","last_clean":"2024-08-17T17:26:50.984918+0000","last_became_active":"2024-08-17T17:26:42.160414+0000","last_became_peered":"2024-08-17T17:26:42.160414+0000","last_unstale":"2024-08-17T17:26:50.984918+0000","last_undegraded":"2024-08-17T17:26:50.984918+0000","last_fullsized":"2024-08-17T17:26:50.984918+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":3699,"log_dups_size":0,"ondisk_log_size":3699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132612096,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2024,"num_write_kb":129504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247469+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:26:48.247469+0000","last_peered":"2024-08-17T17:26:48.247469+0000","last_clean":"2024-08-17T17:26:48.247469+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:26:48.247469+0000","last_undegraded":"2024-08-17T17:26:48.247469+0000","last_fullsized":"2024-08-17T17:26:48.247469+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":150,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247602+0000","last_change":"2024-08-17T17:22:35.164486+0000","last_active":"2024-08-17T17:26:48.247602+0000","last_peered":"2024-08-17T17:26:48.247602+0000","last_clean":"2024-08-17T17:26:48.247602+0000","last_became_active":"2024-08-17T17:22:35.154159+0000","last_became_peered":"2024-08-17T17:22:35.154159+0000","last_unstale":"2024-08-17T17:26:48.247602+0000","last_undegraded":"2024-08-17T17:26:48.247602+0000","last_fullsized":"2024-08-17T17:26:48.247602+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:01.296901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":233,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.190871+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:26:49.190871+0000","last_peered":"2024-08-17T17:26:49.190871+0000","last_clean":"2024-08-17T17:26:49.190871+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:26:49.190871+0000","last_undegraded":"2024-08-17T17:26:49.190871+0000","last_fullsized":"2024-08-17T17:26:49.190871+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":222,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.191662+0000","last_change":"2024-08-17T17:18:55.498226+0000","last_active":"2024-08-17T17:26:49.191662+0000","last_peered":"2024-08-17T17:26:49.191662+0000","last_clean":"2024-08-17T17:26:49.191662+0000","last_became_active":"2024-08-17T17:18:55.497814+0000","last_became_peered":"2024-08-17T17:18:55.497814+0000","last_unstale":"2024-08-17T17:26:49.191662+0000","last_undegraded":"2024-08-17T17:26:49.191662+0000","last_fullsized":"2024-08-17T17:26:49.191662+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:15.742288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-17T17:26:48.247234+0000","last_change":"2024-08-17T17:22:21.209889+0000","last_active":"2024-08-17T17:26:48.247234+0000","last_peered":"2024-08-17T17:26:48.247234+0000","last_clean":"2024-08-17T17:26:48.247234+0000","last_became_active":"2024-08-17T17:22:21.209573+0000","last_became_peered":"2024-08-17T17:22:21.209573+0000","last_unstale":"2024-08-17T17:26:48.247234+0000","last_undegraded":"2024-08-17T17:26:48.247234+0000","last_fullsized":"2024-08-17T17:26:48.247234+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:30.828867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":224,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-17T17:26:49.137060+0000","last_change":"2024-08-17T17:22:35.162817+0000","last_active":"2024-08-17T17:26:49.137060+0000","last_peered":"2024-08-17T17:26:49.137060+0000","last_clean":"2024-08-17T17:26:49.137060+0000","last_became_active":"2024-08-17T17:22:35.145691+0000","last_became_peered":"2024-08-17T17:22:35.145691+0000","last_unstale":"2024-08-17T17:26:49.137060+0000","last_undegraded":"2024-08-17T17:26:49.137060+0000","last_fullsized":"2024-08-17T17:26:49.137060+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:48.293392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":4377149440,"num_objects":66790,"num_object_clones":0,"num_object_copies":200370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3550,"num_objects_unfound":0,"num_objects_dirty":66790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":66790,"num_write_kb":4274560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13095075840,"data_stored":13095075840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100181,"ondisk_log_size":100181,"up":78,"acting":77,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201560,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1988760,"kb_used_data":1791052,"kb_used_omap":1,"kb_used_meta":197630,"kb_avail":91756392,"statfs":{"total":95995035648,"available":93958545408,"internally_reserved":0,"allocated":1834037248,"data_stored":1833020178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":202373579},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463198,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673484,"kb_used_data":1520132,"kb_used_omap":1,"kb_used_meta":153342,"kb_avail":92071668,"statfs":{"total":95995035648,"available":94281388032,"internally_reserved":0,"allocated":1556615168,"data_stored":1555606480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":157022663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724834,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986470,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584588,"kb_used_data":1300804,"kb_used_omap":1,"kb_used_meta":283774,"kb_avail":92160564,"statfs":{"total":95995035648,"available":94372417536,"internally_reserved":0,"allocated":1332023296,"data_stored":1331013812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290585030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435362,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2898684,"kb_used_data":2700404,"kb_used_omap":5,"kb_used_meta":198266,"kb_avail":90846468,"statfs":{"total":95995035648,"available":93026783232,"internally_reserved":0,"allocated":2765213696,"data_stored":2764215866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":203024457},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598888,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2977468,"kb_used_data":2648692,"kb_used_omap":7,"kb_used_meta":328760,"kb_avail":90767684,"statfs":{"total":95995035648,"available":92946108416,"internally_reserved":0,"allocated":2712260608,"data_stored":2711263234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":336650413},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762415,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1808756,"kb_used_data":1632684,"kb_used_omap":7,"kb_used_meta":176056,"kb_avail":91936396,"statfs":{"total":95995035648,"available":94142869504,"internally_reserved":0,"allocated":1671868416,"data_stored":1670878364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":180281761},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925941,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1392072,"kb_used_data":1085504,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":92353080,"statfs":{"total":95995035648,"available":94569553920,"internally_reserved":0,"allocated":1111556096,"data_stored":1110547258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":313909852},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1109983232,"data_stored":1109983232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1703608320,"data_stored":1703608320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2730426368,"data_stored":2730426368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2793603072,"data_stored":2793603072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1337065472,"data_stored":1337065472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1580924928,"data_stored":1580924928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1839464448,"data_stored":1839464448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:27:00.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_1 pgp_num 26 2024-08-17T17:27:00.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:00.535 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:00.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:00 smithi064 ceph-mon[104516]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 96 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 927 op/s; 2610/211593 objects misplaced (1.234%); 7.3 MiB/s, 117 objects/s recovering 2024-08-17T17:27:00.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:00 smithi064 ceph-mon[104516]: osdmap e186: 8 total, 8 up, 7 in 2024-08-17T17:27:00.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:00 smithi064 ceph-mon[109246]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 96 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 927 op/s; 2610/211593 objects misplaced (1.234%); 7.3 MiB/s, 117 objects/s recovering 2024-08-17T17:27:00.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:00 smithi064 ceph-mon[109246]: osdmap e186: 8 total, 8 up, 7 in 2024-08-17T17:27:00.637 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.659 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:00 smithi175 ceph-mon[101434]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 96 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 927 op/s; 2610/211593 objects misplaced (1.234%); 7.3 MiB/s, 117 objects/s recovering 2024-08-17T17:27:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:00 smithi175 ceph-mon[101434]: osdmap e186: 8 total, 8 up, 7 in 2024-08-17T17:27:00.701 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:27:00.701 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:27:00.701 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:27:00.701 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:27:00.701 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:27:00.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:00.781 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:00.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:01.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 71683 71667 95.2648 0 - 0.0101313 2024-08-17T17:27:01.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:01.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:01.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:01.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:01.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:01.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:01.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:27:01.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:27:01.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[104516]: osdmap e187: 8 total, 8 up, 7 in 2024-08-17T17:27:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:27:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:27:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:01 smithi064 ceph-mon[109246]: osdmap e187: 8 total, 8 up, 7 in 2024-08-17T17:27:01.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:01.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:27:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:27:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:01 smithi175 ceph-mon[101434]: osdmap e187: 8 total, 8 up, 7 in 2024-08-17T17:27:01.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:01.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:01.804 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:01.825 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:01.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:01.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:02.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 72125 72109 93.856 13.8125 0.00188168 0.0105509 2024-08-17T17:27:02.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:02.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:02.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:02.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:02.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:02 smithi175 ceph-mon[101434]: pgmap v567: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s; 6652/213441 objects misplaced (3.117%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:27:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:02 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:02 smithi175 ceph-mon[101434]: osdmap e188: 8 total, 8 up, 7 in 2024-08-17T17:27:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[104516]: pgmap v567: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s; 6652/213441 objects misplaced (3.117%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:27:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[104516]: osdmap e188: 8 total, 8 up, 7 in 2024-08-17T17:27:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[109246]: pgmap v567: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s; 6652/213441 objects misplaced (3.117%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:27:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:02 smithi064 ceph-mon[109246]: osdmap e188: 8 total, 8 up, 7 in 2024-08-17T17:27:02.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:02.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:02.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:02.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:03.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 72125 72109 91.9411 0 - 0.0105509 2024-08-17T17:27:03.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:03.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:03.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:03.232 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:03.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:03.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:03.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:03.608 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:03.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:03.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:27:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:27:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:03 smithi175 ceph-mon[101434]: osdmap e189: 8 total, 8 up, 7 in 2024-08-17T17:27:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:27:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:27:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[104516]: osdmap e189: 8 total, 8 up, 7 in 2024-08-17T17:27:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:27:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:27:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-mon[109246]: osdmap e189: 8 total, 8 up, 7 in 2024-08-17T17:27:03.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:03.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:03.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:03.905 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:04.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:04.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 72597 72581 90.6926 14.75 0.00247227 0.0109408 2024-08-17T17:27:04.031 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:04.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:04.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:04.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:04.280 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:03] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.51.0" 2024-08-17T17:27:04.287 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:04.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:04.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:04.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:04.543 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:04.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:04.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:04 smithi175 ceph-mon[101434]: pgmap v570: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 6652/214215 objects misplaced (3.105%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:27:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:04 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:04 smithi175 ceph-mon[101434]: osdmap e190: 8 total, 8 up, 7 in 2024-08-17T17:27:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:04.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[104516]: pgmap v570: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 6652/214215 objects misplaced (3.105%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:27:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[104516]: osdmap e190: 8 total, 8 up, 7 in 2024-08-17T17:27:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[109246]: pgmap v570: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 95 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 14 MiB/s wr, 218 op/s; 6652/214215 objects misplaced (3.105%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:27:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:04 smithi064 ceph-mon[109246]: osdmap e190: 8 total, 8 up, 7 in 2024-08-17T17:27:04.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:04.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:04.892 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:04.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:05.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:05.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 72668 72652 89.0018 4.4375 0.00224031 0.0109347 2024-08-17T17:27:05.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:05.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:05.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:05.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:05.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:05.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:05.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.559 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.660 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:05.683 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:05.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:05.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:05.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:05.950 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 73389 73373 88.1569 45.0625 0.00226096 0.0113114 2024-08-17T17:27:06.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:06.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:06.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.224 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:06.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:06.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:06.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:06 smithi175 ceph-mon[101434]: pgmap v573: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 275 op/s; 6796/216747 objects degraded (3.135%); 10461/216747 objects misplaced (4.826%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:27:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:06 smithi175 ceph-mon[101434]: osdmap e191: 8 total, 8 up, 7 in 2024-08-17T17:27:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:06 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 6796/216747 objects degraded (3.135%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[104516]: pgmap v573: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 275 op/s; 6796/216747 objects degraded (3.135%); 10461/216747 objects misplaced (4.826%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:27:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[104516]: osdmap e191: 8 total, 8 up, 7 in 2024-08-17T17:27:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 6796/216747 objects degraded (3.135%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[109246]: pgmap v573: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 275 op/s; 6796/216747 objects degraded (3.135%); 10461/216747 objects misplaced (4.826%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:27:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[109246]: osdmap e191: 8 total, 8 up, 7 in 2024-08-17T17:27:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:06 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 6796/216747 objects degraded (3.135%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:06.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:06.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:06.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:06.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:06.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:06.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:06.877 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:06.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:07.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 74340 74324 87.615 59.4375 0.0173185 0.0114076 2024-08-17T17:27:07.032 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:07.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:07.134 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:07.160 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:07.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:07.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:07.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:07.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:07.510 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:07.536 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:07.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:07.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:07.892 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:07.920 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:08.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:08.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 75297 75281 87.0993 59.8125 0.00796928 0.011475 2024-08-17T17:27:08.070 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:08.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:08.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:08.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:08.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:08 smithi175 ceph-mon[101434]: pgmap v575: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 9035/218262 objects degraded (4.140%); 10461/218262 objects misplaced (4.793%); 23 MiB/s, 360 objects/s recovering 2024-08-17T17:27:08.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:08.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:08 smithi064 ceph-mon[104516]: pgmap v575: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 9035/218262 objects degraded (4.140%); 10461/218262 objects misplaced (4.793%); 23 MiB/s, 360 objects/s recovering 2024-08-17T17:27:08.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:08 smithi064 ceph-mon[109246]: pgmap v575: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 9035/218262 objects degraded (4.140%); 10461/218262 objects misplaced (4.793%); 23 MiB/s, 360 objects/s recovering 2024-08-17T17:27:08.473 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:08.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.623 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:08.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:08.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:08.751 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:08.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:08.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:08.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.019 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:09.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 76419 76403 86.7907 70.125 0.0233441 0.0115161 2024-08-17T17:27:09.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:09.263 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:09.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:09.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:09.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:09 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:09.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:09 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:09.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:09.557 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:09.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:09 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:09.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:09.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:09.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:10.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 77172 77156 86.0814 47.0625 0.00270965 0.0114976 2024-08-17T17:27:10.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:10.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:10.174 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:10.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:10.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[104516]: pgmap v576: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 525 op/s; 9035/223671 objects degraded (4.039%); 8795/223671 objects misplaced (3.932%); 29 MiB/s, 465 objects/s recovering 2024-08-17T17:27:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[104516]: 6.4 scrub starts 2024-08-17T17:27:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[104516]: osdmap e192: 8 total, 8 up, 7 in 2024-08-17T17:27:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[109246]: pgmap v576: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 525 op/s; 9035/223671 objects degraded (4.039%); 8795/223671 objects misplaced (3.932%); 29 MiB/s, 465 objects/s recovering 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[109246]: 6.4 scrub starts 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[109246]: osdmap e192: 8 total, 8 up, 7 in 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:10.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:10.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:10.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:10.617 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:10 smithi175 ceph-mon[101434]: pgmap v576: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 525 op/s; 9035/223671 objects degraded (4.039%); 8795/223671 objects misplaced (3.932%); 29 MiB/s, 465 objects/s recovering 2024-08-17T17:27:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:10 smithi175 ceph-mon[101434]: 6.4 scrub starts 2024-08-17T17:27:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:10 smithi175 ceph-mon[101434]: osdmap e192: 8 total, 8 up, 7 in 2024-08-17T17:27:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:10.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.858 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:10.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:10.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:11.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.031 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 77691 77675 85.1404 32.4375 0.0179604 0.0117403 2024-08-17T17:27:11.132 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:11.154 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:11.256 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:11.281 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:11.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.429 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:11.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:11 smithi064 ceph-mon[104516]: 6.1 scrub starts 2024-08-17T17:27:11.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:11 smithi064 ceph-mon[104516]: osdmap e193: 8 total, 8 up, 7 in 2024-08-17T17:27:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:11 smithi064 ceph-mon[109246]: 6.1 scrub starts 2024-08-17T17:27:11.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:11 smithi064 ceph-mon[109246]: osdmap e193: 8 total, 8 up, 7 in 2024-08-17T17:27:11.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:11.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:11.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:11.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:11 smithi175 ceph-mon[101434]: 6.1 scrub starts 2024-08-17T17:27:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:11 smithi175 ceph-mon[101434]: osdmap e193: 8 total, 8 up, 7 in 2024-08-17T17:27:11.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:11.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:11.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:12.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 78720 78704 84.7813 64.3125 0.00893744 0.0117898 2024-08-17T17:27:12.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:12.094 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:12.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:12.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:12.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:12.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:12.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:12.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:12.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:12.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:12 smithi175 ceph-mon[101434]: pgmap v579: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s; 9035/226455 objects degraded (3.990%); 8257/226455 objects misplaced (3.646%); 14 MiB/s, 224 objects/s recovering 2024-08-17T17:27:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:12 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9035/226455 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:12 smithi175 ceph-mon[101434]: 6.10 scrub starts 2024-08-17T17:27:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.663 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:12.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[104516]: pgmap v579: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s; 9035/226455 objects degraded (3.990%); 8257/226455 objects misplaced (3.646%); 14 MiB/s, 224 objects/s recovering 2024-08-17T17:27:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9035/226455 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[104516]: 6.10 scrub starts 2024-08-17T17:27:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[109246]: pgmap v579: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s; 9035/226455 objects degraded (3.990%); 8257/226455 objects misplaced (3.646%); 14 MiB/s, 224 objects/s recovering 2024-08-17T17:27:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9035/226455 objects degraded (3.990%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:12 smithi064 ceph-mon[109246]: 6.10 scrub starts 2024-08-17T17:27:12.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:12.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:12.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.919 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:12.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:13.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:13.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 79721 79705 84.4047 62.5625 0.008958 0.0118415 2024-08-17T17:27:13.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:13.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:13.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:13.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:13.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:13.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:13.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:13.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:13.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:13.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:13.539 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:13.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:13.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:13.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:13.683 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:13.785 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:13.809 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:13.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:13.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:14.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:27:14.030395+0000 min lat: 0.00132797 max lat: 2.46805 avg lat: 0.0119131 2024-08-17T17:27:14.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:14.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 80521 80505 83.8313 50 0.00918113 0.0119131 2024-08-17T17:27:14.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:14.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:14.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:14.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:14.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:14.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:14.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:13] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:27:14.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:14 smithi064 ceph-mon[104516]: pgmap v580: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 9035/229686 objects degraded (3.934%); 8257/229686 objects misplaced (3.595%); 11 MiB/s, 174 objects/s recovering 2024-08-17T17:27:14.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:14 smithi064 ceph-mon[109246]: pgmap v580: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 9035/229686 objects degraded (3.934%); 8257/229686 objects misplaced (3.595%); 11 MiB/s, 174 objects/s recovering 2024-08-17T17:27:14.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:14.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:14.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:14.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:14 smithi175 ceph-mon[101434]: pgmap v580: 99 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 9035/229686 objects degraded (3.934%); 8257/229686 objects misplaced (3.595%); 11 MiB/s, 174 objects/s recovering 2024-08-17T17:27:14.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:14.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:14.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:14.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:15.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:15.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 81553 81537 83.5144 64.5 0.018935 0.0119664 2024-08-17T17:27:15.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:15.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:15.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:15.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:15.302 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:15.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:15.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:15.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:15 smithi175 ceph-mon[101434]: 6.4 scrub starts 2024-08-17T17:27:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:15 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:15.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:15 smithi064 ceph-mon[104516]: 6.4 scrub starts 2024-08-17T17:27:15.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:15 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:15.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:15 smithi064 ceph-mon[109246]: 6.4 scrub starts 2024-08-17T17:27:15.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:15 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:15.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:15.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:15.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 82649 82633 83.2721 68.5 0.040317 0.0119957 2024-08-17T17:27:16.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:16.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:16.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:16.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:16.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:16.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:16.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:16.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:16.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:16.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:16.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:16.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:16.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:16 smithi175 ceph-mon[101434]: pgmap v581: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 9035/238425 objects degraded (3.789%); 7376/238425 objects misplaced (3.094%); 16 MiB/s, 249 objects/s recovering 2024-08-17T17:27:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:16 smithi175 ceph-mon[101434]: 6.1 scrub starts 2024-08-17T17:27:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:16 smithi064 ceph-mon[104516]: pgmap v581: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 9035/238425 objects degraded (3.789%); 7376/238425 objects misplaced (3.094%); 16 MiB/s, 249 objects/s recovering 2024-08-17T17:27:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:16 smithi064 ceph-mon[104516]: 6.1 scrub starts 2024-08-17T17:27:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:16 smithi064 ceph-mon[109246]: pgmap v581: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 9035/238425 objects degraded (3.789%); 7376/238425 objects misplaced (3.094%); 16 MiB/s, 249 objects/s recovering 2024-08-17T17:27:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:16 smithi064 ceph-mon[109246]: 6.1 scrub starts 2024-08-17T17:27:16.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:16.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:16.865 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:16.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:16.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:17.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:17.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:17.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 83666 83650 82.9593 63.5625 0.0270038 0.0120483 2024-08-17T17:27:17.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:27:17.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:17.179 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:17.281 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:17.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:17.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:17.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:17.431 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:17.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:17.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:17.557 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:17.623 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:17 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9035/243081 objects degraded (3.717%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:17.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:17 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9035/243081 objects degraded (3.717%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:17.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:17.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:17 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9035/243081 objects degraded (3.717%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:17.716 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:17.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:17.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:17.773 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:17.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:17.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:17.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:18.031 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 84674 84658 82.6474 63 0.0227456 0.0120944 2024-08-17T17:27:18.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.047 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:18.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:18.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:18.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:18.499 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:18.601 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:18.625 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:18.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:18 smithi064 ceph-mon[104516]: pgmap v582: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 9035/243081 objects degraded (3.717%); 7376/243081 objects misplaced (3.034%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:27:18.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:18 smithi064 ceph-mon[104516]: 6.10 scrub starts 2024-08-17T17:27:18.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:18 smithi064 ceph-mon[109246]: pgmap v582: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 9035/243081 objects degraded (3.717%); 7376/243081 objects misplaced (3.034%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:27:18.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:18 smithi064 ceph-mon[109246]: 6.10 scrub starts 2024-08-17T17:27:18.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:18 smithi175 ceph-mon[101434]: pgmap v582: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 9035/243081 objects degraded (3.717%); 7376/243081 objects misplaced (3.034%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:27:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:18 smithi175 ceph-mon[101434]: 6.10 scrub starts 2024-08-17T17:27:18.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:18.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:18.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:19.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.031 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 85798 85782 82.4558 70.25 0.00733444 0.0121213 2024-08-17T17:27:19.061 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:19.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:19.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:19.305 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:19.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:19.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:19.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:19.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:19.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:19.738 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:19.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:19.881 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:19.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.008 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:20.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:20.031 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 86878 86862 82.2291 67.5 0.0235451 0.0121548 2024-08-17T17:27:20.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:20.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:20.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:20.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:20.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:20.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:20.544 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:20.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:20 smithi175 ceph-mon[101434]: pgmap v583: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s; 9035/250776 objects degraded (3.603%); 6934/250776 objects misplaced (2.765%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:20.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:20 smithi064 ceph-mon[104516]: pgmap v583: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s; 9035/250776 objects degraded (3.603%); 6934/250776 objects misplaced (2.765%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:20 smithi064 ceph-mon[109246]: pgmap v583: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s; 9035/250776 objects degraded (3.603%); 6934/250776 objects misplaced (2.765%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:20.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:20.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:20.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:20.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:20.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:21.031 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 87743 87727 81.8087 54.0625 0.00442819 0.012164 2024-08-17T17:27:21.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:21.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:21.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:21.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:21.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:21.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:21.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:21.497 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:21.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:21.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:21 smithi175 ceph-mon[101434]: 6.4 scrub starts 2024-08-17T17:27:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:21 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:21 smithi175 ceph-mon[101434]: osdmap e194: 8 total, 8 up, 7 in 2024-08-17T17:27:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[104516]: 6.4 scrub starts 2024-08-17T17:27:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[104516]: osdmap e194: 8 total, 8 up, 7 in 2024-08-17T17:27:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[109246]: 6.4 scrub starts 2024-08-17T17:27:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:21 smithi064 ceph-mon[109246]: osdmap e194: 8 total, 8 up, 7 in 2024-08-17T17:27:21.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:21.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:21.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:21.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:21.919 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:22.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:22.033 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 88606 88590 81.3968 53.9375 0.00954989 0.0122802 2024-08-17T17:27:22.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:22.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:22.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:22.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:22.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:22.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:22.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:22.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:22.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:22 smithi175 ceph-mon[101434]: pgmap v585: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 9035/254814 objects degraded (3.546%); 6490/254814 objects misplaced (2.547%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:27:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:22 smithi175 ceph-mon[101434]: osdmap e195: 8 total, 8 up, 7 in 2024-08-17T17:27:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[104516]: pgmap v585: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 9035/254814 objects degraded (3.546%); 6490/254814 objects misplaced (2.547%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:27:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[104516]: osdmap e195: 8 total, 8 up, 7 in 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[109246]: pgmap v585: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 9035/254814 objects degraded (3.546%); 6490/254814 objects misplaced (2.547%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[109246]: osdmap e195: 8 total, 8 up, 7 in 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:22.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:22.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:22.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:22.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:23.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:23.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:23.033 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 89689 89673 81.1981 67.6875 0.0128585 0.0123104 2024-08-17T17:27:23.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:23.173 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:23.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:23.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:23.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:23.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:23.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:23.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:23.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9035/254814 objects degraded (3.546%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:23.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9035/254814 objects degraded (3.546%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9035/254814 objects degraded (3.546%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:23.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:23.761 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:23.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:23.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:23.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:24.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:24.033 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 90815 90799 81.0434 70.375 0.0119019 0.0123334 2024-08-17T17:27:24.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:24.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:24.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:24.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:24.322 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:23] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:27:24.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:24.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:24.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:24.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[104516]: pgmap v587: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 6796/258807 objects degraded (2.626%); 8360/258807 objects misplaced (3.230%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:27:24.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:24.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:24.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:24.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[109246]: pgmap v587: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 6796/258807 objects degraded (2.626%); 8360/258807 objects misplaced (3.230%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:27:24.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:24.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:24.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:24.580 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:24 smithi175 ceph-mon[101434]: pgmap v587: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 6796/258807 objects degraded (2.626%); 8360/258807 objects misplaced (3.230%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:27:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:24.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:24.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:24.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:24.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:24.864 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:24.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:24.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:25.019 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:25.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 91851 91835 80.8136 64.75 0.00682764 0.012368 2024-08-17T17:27:25.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:25.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:25.244 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:27:25.244 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:27:25.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:27:25.244 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131915 5.c 5:365a0339:::benchmark_data_smithi064_147470_object131914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 323.20824784299998, 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1561467159999999, 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:27:25.245 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131915 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031610+0000", 2024-08-17T17:27:25.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031611+0000", 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6780000000000001e-06 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031633+0000", 2024-08-17T17:27:25.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1382e-05 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031647+0000", 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4217e-05 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031653+0000", 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8119999999999996e-06 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031715+0000", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2317000000000007e-05 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.031889+0000", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173826 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.032068+0000", 2024-08-17T17:27:25.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017936499999999999 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050025+0000", 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017957101 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050038+0000", 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3128e-05 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049267+0000", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.999229005 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049283+0000", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5880999999999999e-05 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.187675+0000", 2024-08-17T17:27:25.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.138391844 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.187756+0000", 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1160000000000002e-05 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131927 5.c 5:3e0315ab:::benchmark_data_smithi064_147470_object131926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:27:25.252 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 323.17570369200001, 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1700607590000001, 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131927 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064154+0000", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064156+0000", 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4099999999999998e-06 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064196+0000", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9903000000000001e-05 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064219+0000", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2807e-05 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064229+0000", 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0387999999999999e-05 2024-08-17T17:27:25.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064439+0000", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021010700000000001 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.064462+0000", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2501999999999999e-05 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049376+0000", 2024-08-17T17:27:25.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98491408300000005 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049385+0000", 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.188213+0000", 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1388277579999999 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.188382+0000", 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016944599999999999 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.189048+0000", 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066579400000000004 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203168+0000", 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014119567 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.207367+0000", 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041998019999999999 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.218319+0000", 2024-08-17T17:27:25.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010952147000000001 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234010+0000", 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015690763999999999 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234083+0000", 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3046999999999994e-05 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234214+0000", 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013113400000000001 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131930 5.c 5:32de370a:::benchmark_data_smithi064_147470_object131929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:27:25.262 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 323.17236071999997, 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1672973080000002, 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131930 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067497+0000", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067498+0000", 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.747e-06 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067526+0000", 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7480000000000001e-05 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067544+0000", 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8377999999999999e-05 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067553+0000", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8380000000000008e-06 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067608+0000", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4829e-05 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.067625+0000", 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6599000000000002e-05 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049418+0000", 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98179379899999997 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.049428+0000", 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0128e-05 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203205+0000", 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1537763000000001 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203366+0000", 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016128200000000001 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.203557+0000", 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019087600000000001 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.207352+0000", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037946690000000001 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.215942+0000", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085902219999999998 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.218428+0000", 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024865429999999999 2024-08-17T17:27:25.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234619+0000", 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016191082999999998 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234679+0000", 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9117999999999999e-05 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.234794+0000", 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115417 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70414 6.11 6:8b44c908:::benchmark_data_smithi064_185391_object70413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.209690+0000", 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.030167117000001, 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1930821680000001, 2024-08-17T17:27:25.271 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70414 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209690+0000", 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209690+0000", 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209691+0000", 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4200000000000001e-07 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209732+0000", 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1282999999999998e-05 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209742+0000", 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0366999999999999e-05 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209749+0000", 2024-08-17T17:27:25.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2670000000000003e-06 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211015+0000", 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001266742 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211022+0000", 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3239999999999999e-06 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.359600+0000", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1485785449999999 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.359763+0000", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000163074 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.359964+0000", 2024-08-17T17:27:25.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020095599999999999 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.363333+0000", 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033690619999999999 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.363427+0000", 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3678999999999995e-05 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378122+0000", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014695035 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.401314+0000", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023192332999999999 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.401389+0000", 2024-08-17T17:27:25.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4715000000000005e-05 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.402772+0000", 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013833439999999999 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70443 6.11 6:89fc7e60:::benchmark_data_smithi064_185391_object70442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.209757+0000", 2024-08-17T17:27:25.279 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.030100314999999, 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1942969400000001, 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70443 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209757+0000", 2024-08-17T17:27:25.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209757+0000", 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209757+0000", 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2300000000000002e-07 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209799+0000", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1572000000000001e-05 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209807+0000", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6820000000000003e-06 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209812+0000", 2024-08-17T17:27:25.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7019999999999998e-06 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211045+0000", 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012326570000000001 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211052+0000", 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4280000000000001e-06 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.361592+0000", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1505403350000001 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.361747+0000", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155618 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.361933+0000", 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000185973 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.366240+0000", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043070080000000002 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.366288+0000", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7261e-05 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378179+0000", 2024-08-17T17:27:25.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011891439 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.402831+0000", 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024651482999999998 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.402907+0000", 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6234000000000002e-05 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.404054+0000", 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011471249999999999 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70474 6.11 6:8bf87045:::benchmark_data_smithi064_185391_object70473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.209820+0000", 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.030037179000001, 2024-08-17T17:27:25.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1955486980000001, 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70474 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209820+0000", 2024-08-17T17:27:25.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209820+0000", 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209821+0000", 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0999999999999999e-07 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209866+0000", 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5905000000000002e-05 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209874+0000", 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0960000000000003e-06 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209879+0000", 2024-08-17T17:27:25.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6420000000000004e-06 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211101+0000", 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001222108 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211108+0000", 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5060000000000001e-06 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.363461+0000", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1523535330000001 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.363619+0000", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015754399999999999 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.363795+0000", 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017647100000000001 2024-08-17T17:27:25.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.366305+0000", 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025097650000000002 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368287+0000", 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019822939999999999 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378242+0000", 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099548320000000003 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.404122+0000", 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025880170000000001 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.404180+0000", 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7305000000000001e-05 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.405369+0000", 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001189117 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70509 6.11 6:8964720d:::benchmark_data_smithi064_185391_object70508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.209896+0000", 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029961835000002, 2024-08-17T17:27:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.196633788, 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70509 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209896+0000", 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209896+0000", 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.209896+0000", 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6800000000000001e-07 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210042+0000", 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145916 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210051+0000", 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5880000000000004e-06 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210059+0000", 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6490000000000008e-06 2024-08-17T17:27:25.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211132+0000", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001072733 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211138+0000", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5710000000000003e-06 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.364958+0000", 2024-08-17T17:27:25.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1538194980000001 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.365096+0000", 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137582 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.365256+0000", 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000160669 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368376+0000", 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003119461 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368416+0000", 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0593999999999998e-05 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.381655+0000", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013238901000000001 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.405427+0000", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023771934000000002 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.405491+0000", 2024-08-17T17:27:25.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4406000000000006e-05 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.406529+0000", 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001037818 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70510 6.11 6:8fb5af88:::benchmark_data_smithi064_185391_object70509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.303 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210069+0000", 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029788523000001, 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1975646580000001, 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70510 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210069+0000", 2024-08-17T17:27:25.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210069+0000", 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210069+0000", 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5799999999999999e-07 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210116+0000", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7030000000000002e-05 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210124+0000", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1899999999999998e-06 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210129+0000", 2024-08-17T17:27:25.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4099999999999999e-06 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211165+0000", 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010359810000000001 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211171+0000", 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0039999999999996e-06 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368452+0000", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1572806419999999 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368616+0000", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016439699999999999 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.368786+0000", 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016949599999999999 2024-08-17T17:27:25.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377693+0000", 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089076899999999994 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377715+0000", 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1662e-05 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.381814+0000", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040990330000000002 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.406606+0000", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024791865 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.406663+0000", 2024-08-17T17:27:25.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6817999999999997e-05 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.407633+0000", 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097088199999999995 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70573 6.11 6:8a06845b:::benchmark_data_smithi064_185391_object70572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210143+0000", 2024-08-17T17:27:25.312 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029714846000001, 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1989634760000001, 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70573 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210143+0000", 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210143+0000", 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210143+0000", 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2199999999999999e-07 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210186+0000", 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2862999999999999e-05 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210195+0000", 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4269999999999997e-06 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210201+0000", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6370000000000004e-06 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211209+0000", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010083539999999999 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211214+0000", 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8840000000000002e-06 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.370066+0000", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1588519939999999 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.370242+0000", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000176081 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.370656+0000", 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041365800000000002 2024-08-17T17:27:25.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377724+0000", 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070678670000000002 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377847+0000", 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012310199999999999 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391293+0000", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013445741000000001 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.407698+0000", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016405795000000001 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.407764+0000", 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6157999999999999e-05 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.409106+0000", 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001341493 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70582 6.11 6:8fb2fc1f:::benchmark_data_smithi064_185391_object70581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210209+0000", 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029647895, 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2006917640000001, 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.320 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70582 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210209+0000", 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210209+0000", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210210+0000", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4999999999999998e-07 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210257+0000", 2024-08-17T17:27:25.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6601999999999999e-05 2024-08-17T17:27:25.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210264+0000", 2024-08-17T17:27:25.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9659999999999996e-06 2024-08-17T17:27:25.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210270+0000", 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7030000000000003e-06 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211271+0000", 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010012580000000001 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211279+0000", 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9850000000000006e-06 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.373417+0000", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.162137486 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.373586+0000", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016886200000000001 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.373864+0000", 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027812399999999998 2024-08-17T17:27:25.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377865+0000", 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040008350000000003 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.377995+0000", 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013042699999999999 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391369+0000", 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013373612999999999 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.409167+0000", 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017798703999999999 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.409228+0000", 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0448000000000001e-05 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.410901+0000", 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016733010000000001 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70588 6.11 6:8985a7cf:::benchmark_data_smithi064_185391_object70587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210278+0000", 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029579545000001, 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.204279637, 2024-08-17T17:27:25.330 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70588 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210278+0000", 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210278+0000", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210278+0000", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7e-07 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210325+0000", 2024-08-17T17:27:25.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6978000000000003e-05 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210333+0000", 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8499999999999994e-06 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210339+0000", 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5600000000000001e-06 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211325+0000", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098674000000000006 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211334+0000", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4540000000000007e-06 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378291+0000", 2024-08-17T17:27:25.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1669566549999999 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378467+0000", 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017636799999999999 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.378639+0000", 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017253 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.381723+0000", 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030831629999999999 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.390947+0000", 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092244820000000009 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391430+0000", 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048321300000000002 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.410953+0000", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019522741999999999 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.411017+0000", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4307999999999995e-05 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.414557+0000", 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003540124 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.337 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70627 6.11 6:8a571d8f:::benchmark_data_smithi064_185391_object70626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210345+0000", 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029511924000001, 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.205699072, 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.338 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70627 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210345+0000", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210345+0000", 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210346+0000", 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7100000000000002e-07 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210387+0000", 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1236999999999998e-05 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210395+0000", 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2800000000000003e-06 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210401+0000", 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.22e-06 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211371+0000", 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097031899999999998 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211378+0000", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6309999999999997e-06 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.379860+0000", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.168482775 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.380004+0000", 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143215 2024-08-17T17:27:25.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.380172+0000", 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016851299999999999 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.390995+0000", 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010822531 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391012+0000", 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7124000000000001e-05 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391489+0000", 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047685400000000003 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.414620+0000", 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023130992999999999 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.414683+0000", 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3393999999999996e-05 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.416045+0000", 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001361515 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.345 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70682 6.11 6:8fe0cb43:::benchmark_data_smithi064_185391_object70681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:56.210407+0000", 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.029450227000002, 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.207084738, 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70682 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.346 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210407+0000", 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210407+0000", 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210408+0000", 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.27e-07 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210451+0000", 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3115000000000003e-05 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210459+0000", 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.210465+0000", 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7409999999999998e-06 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211405+0000", 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093974300000000005 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.211409+0000", 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2300000000000002e-06 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.381855+0000", 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1704465310000001 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.382102+0000", 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000246698 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.382294+0000", 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000192 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391019+0000", 2024-08-17T17:27:25.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087244769999999996 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391076+0000", 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7865000000000002e-05 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.391524+0000", 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000447305 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.416101+0000", 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024576958999999999 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.416163+0000", 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2133999999999998e-05 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:57.417492+0000", 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001328975 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71906 6.14 6:2faac078:::benchmark_data_smithi064_185391_object71905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.150480+0000", 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.089376876999999, 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2439831939999999, 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.355 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71906 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150480+0000", 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150480+0000", 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150481+0000", 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0699999999999997e-07 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150703+0000", 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000221516 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150712+0000", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4620000000000002e-06 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150719+0000", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.827e-06 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150759+0000", 2024-08-17T17:27:25.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0444000000000001e-05 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150767+0000", 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.362e-06 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.318959+0000", 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.16819224 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.319117+0000", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157925 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.319297+0000", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018047399999999999 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.322225+0000", 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029281860000000002 2024-08-17T17:27:25.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.335723+0000", 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013497510000000001 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.337255+0000", 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001532519 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.392717+0000", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.055461923000000003 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.392776+0000", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8167000000000001e-05 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.394464+0000", 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001688132 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72002 6.14 6:2f5aaa49:::benchmark_data_smithi064_185391_object72001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.150735+0000", 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.089122533000001, 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.245297383, 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.363 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72002 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150735+0000", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150735+0000", 2024-08-17T17:27:25.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150735+0000", 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3900000000000004e-07 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150781+0000", 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6034e-05 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150790+0000", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4649999999999994e-06 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150796+0000", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0959999999999997e-06 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150833+0000", 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7333999999999999e-05 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150840+0000", 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.002e-06 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.320640+0000", 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.169799112 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.320781+0000", 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141762 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.320954+0000", 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000172762 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.322372+0000", 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001418152 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.337299+0000", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014926974000000001 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.340495+0000", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031959979999999998 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.394524+0000", 2024-08-17T17:27:25.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.054028908 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.394584+0000", 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9539e-05 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.396032+0000", 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014486060000000001 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72049 6.14 6:2f0a57ae:::benchmark_data_smithi064_185391_object72048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.150811+0000", 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.089046613000001, 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.246871581, 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72049 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150811+0000", 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150811+0000", 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150811+0000", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4300000000000003e-07 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150855+0000", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4092999999999999e-05 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150863+0000", 2024-08-17T17:27:25.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8359999999999992e-06 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150869+0000", 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9660000000000001e-06 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150911+0000", 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2067000000000002e-05 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150918+0000", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0929999999999997e-06 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.323968+0000", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173049641 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.324147+0000", 2024-08-17T17:27:25.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017913099999999999 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.324358+0000", 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000211036 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.332549+0000", 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00819048 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.338999+0000", 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064502889999999997 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.343400+0000", 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044007589999999997 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.396082+0000", 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.052682101000000002 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.396144+0000", 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.177e-05 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.397682+0000", 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015387759999999999 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72072 6.14 6:29805d6c:::benchmark_data_smithi064_185391_object72071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.150892+0000", 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.088965513000002, 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.248474257, 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72072 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150892+0000", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150892+0000", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150892+0000", 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.37e-07 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150941+0000", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8763000000000001e-05 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150950+0000", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4479999999999994e-06 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150955+0000", 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4190000000000001e-06 2024-08-17T17:27:25.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150994+0000", 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9261000000000002e-05 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151002+0000", 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2899999999999997e-06 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.326131+0000", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.175129549 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.326270+0000", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013899100000000001 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.326426+0000", 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015617100000000001 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.332556+0000", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061295339999999999 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.340560+0000", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080043159999999992 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.343496+0000", 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002936138 2024-08-17T17:27:25.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.397754+0000", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.054257976999999999 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.397810+0000", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5967000000000001e-05 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.399366+0000", 2024-08-17T17:27:25.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001555896 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72075 6.14 6:29c45fab:::benchmark_data_smithi064_185391_object72074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.150970+0000", 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.088887226000001, 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2501142060000001, 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.386 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72075 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150970+0000", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150970+0000", 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.150971+0000", 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8800000000000003e-07 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151053+0000", 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2601000000000004e-05 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151062+0000", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9579999999999996e-06 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151068+0000", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0530000000000003e-06 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151108+0000", 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0091e-05 2024-08-17T17:27:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151116+0000", 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7060000000000004e-06 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.327701+0000", 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.176585276 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.327832+0000", 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013088499999999999 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.327952+0000", 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012018700000000001 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.332562+0000", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004610014 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.343448+0000", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010885344999999999 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.343602+0000", 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154489 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.399498+0000", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.055895515 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.399552+0000", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4008000000000002e-05 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.401084+0000", 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015325899999999999 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.393 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72090 6.14 6:2f54693d:::benchmark_data_smithi064_185391_object72089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.151083+0000", 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.08877425, 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.251969074, 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72090 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151083+0000", 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151083+0000", 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:25.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151084+0000", 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4399999999999999e-07 2024-08-17T17:27:25.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151111+0000", 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6689000000000001e-05 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151118+0000", 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1450000000000002e-06 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151124+0000", 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1210000000000003e-06 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151160+0000", 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5744999999999997e-05 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151167+0000", 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1359999999999999e-06 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.328943+0000", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.177776157 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.329043+0000", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100609 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.329163+0000", 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011919200000000001 2024-08-17T17:27:25.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.332628+0000", 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034655229999999999 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.343547+0000", 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010919294 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.389525+0000", 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045977668999999999 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.401446+0000", 2024-08-17T17:27:25.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011920922 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.401568+0000", 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122108 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.403052+0000", 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00148402 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:72125 6.14 6:296038a8:::benchmark_data_smithi064_185391_object72124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:02.151137+0000", 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.088719942000001, 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.265385685, 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 72125 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:25.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151137+0000", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151137+0000", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151138+0000", 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2600000000000002e-07 2024-08-17T17:27:25.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151160+0000", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2206999999999999e-05 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151168+0000", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3830000000000003e-06 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151173+0000", 2024-08-17T17:27:25.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7089999999999999e-06 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151213+0000", 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9298000000000002e-05 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:02.151220+0000", 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.391e-06 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.389576+0000", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2383558210000001 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.391057+0000", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001481603 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,7", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.391213+0000", 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015573299999999999 2024-08-17T17:27:25.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.399419+0000", 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082057310000000008 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.399459+0000", 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9969000000000003e-05 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.405374+0000", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059150690000000002 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.413059+0000", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076851690000000004 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.413170+0000", 2024-08-17T17:27:25.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011052600000000001 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:03.416523+0000", 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033535499999999998 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:25.411 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:27:25.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:25.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3501440552' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:25.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:25.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3501440552' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:25.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:25.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:25.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3501440552' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:25.685 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:25.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:25.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:25.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:25.939 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:26.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 92687 92671 80.4169 52.25 0.01798 0.0124297 2024-08-17T17:27:26.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:26.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:26.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:26.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:26.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:26.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:26.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:26.510 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:26.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:26.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:26.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:26 smithi175 ceph-mon[101434]: pgmap v588: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 6796/268884 objects degraded (2.527%); 7770/268884 objects misplaced (2.890%); 10 MiB/s, 165 objects/s recovering 2024-08-17T17:27:26.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:26 smithi064 ceph-mon[104516]: pgmap v588: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 6796/268884 objects degraded (2.527%); 7770/268884 objects misplaced (2.890%); 10 MiB/s, 165 objects/s recovering 2024-08-17T17:27:26.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:26 smithi064 ceph-mon[109246]: pgmap v588: 99 pgs: 2 active+recovering+undersized+remapped, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 6796/268884 objects degraded (2.527%); 7770/268884 objects misplaced (2.890%); 10 MiB/s, 165 objects/s recovering 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:27:26.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:26.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:26.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:26.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:26.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:26.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:26.914 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:27.016 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:27.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 93573 93557 80.0739 55.375 0.00974998 0.0124819 2024-08-17T17:27:27.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:27.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:27.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:27.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:27.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:27.370 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pgp_num to 26 2024-08-17T17:27:27.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:27.452 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:27.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:27.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:27 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2511208500' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:27 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2511208500' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2511208500' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:27 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:27:27.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:27.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:27.857 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:27.959 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:27.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:28.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 94712 94696 79.9538 71.1875 0.0109791 0.0125015 2024-08-17T17:27:28.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:28.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:28.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.440 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:28.580 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:28 smithi175 ceph-mon[101434]: pgmap v589: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 6796/273804 objects degraded (2.482%); 7051/273804 objects misplaced (2.575%); 13 MiB/s, 200 objects/s recovering 2024-08-17T17:27:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:27:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:28 smithi175 ceph-mon[101434]: osdmap e196: 8 total, 8 up, 7 in 2024-08-17T17:27:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6796/273804 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:28.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[104516]: pgmap v589: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 6796/273804 objects degraded (2.482%); 7051/273804 objects misplaced (2.575%); 13 MiB/s, 200 objects/s recovering 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[104516]: osdmap e196: 8 total, 8 up, 7 in 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6796/273804 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[109246]: pgmap v589: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s; 6796/273804 objects degraded (2.482%); 7051/273804 objects misplaced (2.575%); 13 MiB/s, 200 objects/s recovering 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[109246]: osdmap e196: 8 total, 8 up, 7 in 2024-08-17T17:27:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6796/273804 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:28.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:28.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.963 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.994 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:28.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:29.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 95806 95790 79.7993 68.375 0.0161469 0.0125264 2024-08-17T17:27:29.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:29.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:27:29.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:29.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:29.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:29.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:29.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:29.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:29.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:29.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:29.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:29.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:29.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:29.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:29.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:29.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:29.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:30.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:30.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 96903 96887 79.65 68.5625 0.00656521 0.0125498 2024-08-17T17:27:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:30.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:30.195 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:30.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:30.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:30.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:30.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:30.463 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:30.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:30.492 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:30.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:30.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:30 smithi064 ceph-mon[104516]: pgmap v591: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 6796/280887 objects degraded (2.419%); 7051/280887 objects misplaced (2.510%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:27:30.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:30 smithi064 ceph-mon[109246]: pgmap v591: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 6796/280887 objects degraded (2.419%); 7051/280887 objects misplaced (2.510%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:27:30.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:30 smithi175 ceph-mon[101434]: pgmap v591: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 6796/280887 objects degraded (2.419%); 7051/280887 objects misplaced (2.510%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:27:30.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:30.786 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:30.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:30.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:30.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:31.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:31.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 97867 97851 79.398 60.25 0.0122027 0.0125894 2024-08-17T17:27:31.054 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:31.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:31.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:31.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:31.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:31.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:31.380 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:27:31.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:31.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:31.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:31.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1061408887' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:31 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1061408887' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:31 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1061408887' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:31 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:31.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:31.746 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:31.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:31.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:31.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:32.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 99040 99024 79.3199 73.3125 0.0136267 0.0126017 2024-08-17T17:27:32.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:32.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:32.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:32.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:32.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:32.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:32.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:32.547 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:32 smithi175 ceph-mon[101434]: pgmap v592: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s; 6796/284676 objects degraded (2.387%); 6507/284676 objects misplaced (2.286%); 8.4 MiB/s, 133 objects/s recovering 2024-08-17T17:27:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:32 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:32 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:32 smithi175 ceph-mon[101434]: osdmap e197: 8 total, 8 up, 7 in 2024-08-17T17:27:32.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[104516]: pgmap v592: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s; 6796/284676 objects degraded (2.387%); 6507/284676 objects misplaced (2.286%); 8.4 MiB/s, 133 objects/s recovering 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[104516]: osdmap e197: 8 total, 8 up, 7 in 2024-08-17T17:27:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[109246]: pgmap v592: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 947 op/s; 6796/284676 objects degraded (2.387%); 6507/284676 objects misplaced (2.286%); 8.4 MiB/s, 133 objects/s recovering 2024-08-17T17:27:32.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:32.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:32.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:32.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:32 smithi064 ceph-mon[109246]: osdmap e197: 8 total, 8 up, 7 in 2024-08-17T17:27:32.713 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:32.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:32.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:32.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:32.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:33.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 100157 100141 79.1995 69.8125 0.0153159 0.0126209 2024-08-17T17:27:33.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:33.139 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:33.240 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:33.266 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:33.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:33.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:33.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:33.578 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:33.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6796/284676 objects degraded (2.387%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:33 smithi175 ceph-mon[101434]: osdmap e198: 8 total, 8 up, 7 in 2024-08-17T17:27:33.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6796/284676 objects degraded (2.387%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:33 smithi064 ceph-mon[104516]: osdmap e198: 8 total, 8 up, 7 in 2024-08-17T17:27:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6796/284676 objects degraded (2.387%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:33 smithi064 ceph-mon[109246]: osdmap e198: 8 total, 8 up, 7 in 2024-08-17T17:27:33.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:33.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:33.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:33.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:34.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:34.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:27:34.036285+0000 min lat: 0.00132797 max lat: 2.46805 avg lat: 0.0126219 2024-08-17T17:27:34.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:34.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 100737 100721 78.6628 36.25 0.00251275 0.0126219 2024-08-17T17:27:34.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:34.108 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:33] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-08-17T17:27:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.135 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:34.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:34.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:27:34.259 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:34.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:34.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:34.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: pgmap v594: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 6796/289137 objects degraded (2.350%); 5529/289137 objects misplaced (1.912%); 15 MiB/s, 246 objects/s recovering 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: 6.4 scrub starts 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: 6.19 scrub starts 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: 6.2 scrub starts 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: 6.7 scrub starts 2024-08-17T17:27:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:34 smithi175 ceph-mon[101434]: osdmap e199: 8 total, 8 up, 7 in 2024-08-17T17:27:34.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: pgmap v594: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 6796/289137 objects degraded (2.350%); 5529/289137 objects misplaced (1.912%); 15 MiB/s, 246 objects/s recovering 2024-08-17T17:27:34.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: 6.4 scrub starts 2024-08-17T17:27:34.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: 6.19 scrub starts 2024-08-17T17:27:34.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: 6.2 scrub starts 2024-08-17T17:27:34.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: 6.7 scrub starts 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[104516]: osdmap e199: 8 total, 8 up, 7 in 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: pgmap v594: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s; 6796/289137 objects degraded (2.350%); 5529/289137 objects misplaced (1.912%); 15 MiB/s, 246 objects/s recovering 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: 6.4 scrub starts 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: 6.19 scrub starts 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: 6.2 scrub starts 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: 6.7 scrub starts 2024-08-17T17:27:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:34 smithi064 ceph-mon[109246]: osdmap e199: 8 total, 8 up, 7 in 2024-08-17T17:27:34.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:34.749 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:27:34.750 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:27:34.750 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:27:34.750 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:27:34.750 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:27:34.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:34.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:34.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:34.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:35.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 101309 101293 78.133 35.75 0.00272843 0.0127232 2024-08-17T17:27:35.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:35.085 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:34 smithi175 systemd[1]: Stopping Ceph osd.6 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:27:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:35.188 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:35.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:35.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:35.402 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[51336]: 2024-08-17T17:27:35.082+0000 7f7da6e81640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:27:35.403 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[51336]: 2024-08-17T17:27:35.082+0000 7f7da6e81640 -1 osd.6 199 *** Got signal Terminated *** 2024-08-17T17:27:35.403 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[51336]: 2024-08-17T17:27:35.082+0000 7f7da6e81640 -1 osd.6 199 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:27:35.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:35.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:35.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:35.592 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:35.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 podman[143115]: 2024-08-17 17:27:35.511857186 +0000 UTC m=+0.550174412 container died 227f3eb1987a510981ed7d9bf0e58f034e24ed699477c5cb3c8b9d9f40c8649e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0) 2024-08-17T17:27:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: 6.2 scrub ok 2024-08-17T17:27:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: 6.19 scrub ok 2024-08-17T17:27:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: 6.4 scrub ok 2024-08-17T17:27:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: 6.7 scrub ok 2024-08-17T17:27:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3264365645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:35.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: osd.6 marked itself down and dead 2024-08-17T17:27:35.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:27:35.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:35 smithi175 ceph-mon[101434]: osdmap e200: 8 total, 7 up, 7 in 2024-08-17T17:27:35.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: 6.2 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: 6.19 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: 6.4 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: 6.7 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3264365645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: osd.6 marked itself down and dead 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[104516]: osdmap e200: 8 total, 7 up, 7 in 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: 6.2 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: 6.19 scrub ok 2024-08-17T17:27:35.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: 6.4 scrub ok 2024-08-17T17:27:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: 6.7 scrub ok 2024-08-17T17:27:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3264365645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: osd.6 marked itself down and dead 2024-08-17T17:27:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:27:35.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:35 smithi064 ceph-mon[109246]: osdmap e200: 8 total, 7 up, 7 in 2024-08-17T17:27:35.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:35.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:35.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.842 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:35.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:35.966 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:36.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 101360 101344 77.2193 3.1875 0.00225132 0.0128142 2024-08-17T17:27:36.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:36.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:36.161 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 podman[143115]: 2024-08-17 17:27:35.976119383 +0000 UTC m=+1.014436602 container cleanup 227f3eb1987a510981ed7d9bf0e58f034e24ed699477c5cb3c8b9d9f40c8649e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux ) 2024-08-17T17:27:36.161 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:35 smithi175 bash[143115]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:27:36.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:36.232 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:36.334 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:36.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:36.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:36.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:36.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:36.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:36.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:36 smithi175 podman[143149]: 2024-08-17 17:27:36.361713806 +0000 UTC m=+0.845599871 container remove 227f3eb1987a510981ed7d9bf0e58f034e24ed699477c5cb3c8b9d9f40c8649e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:27:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:36 smithi175 ceph-mon[101434]: pgmap v597: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2196/298692 objects degraded (0.735%); 10063/298692 objects misplaced (3.369%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:27:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:36 smithi175 ceph-mon[101434]: 6.11 scrub starts 2024-08-17T17:27:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:36 smithi175 ceph-mon[101434]: 6.11 scrub ok 2024-08-17T17:27:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:36 smithi175 ceph-mon[101434]: osdmap e201: 8 total, 7 up, 7 in 2024-08-17T17:27:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[104516]: pgmap v597: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2196/298692 objects degraded (0.735%); 10063/298692 objects misplaced (3.369%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[104516]: 6.11 scrub starts 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[104516]: 6.11 scrub ok 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[104516]: osdmap e201: 8 total, 7 up, 7 in 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[109246]: pgmap v597: 99 pgs: 3 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 29 MiB/s wr, 457 op/s; 2196/298692 objects degraded (0.735%); 10063/298692 objects misplaced (3.369%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[109246]: 6.11 scrub starts 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[109246]: 6.11 scrub ok 2024-08-17T17:27:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:36 smithi064 ceph-mon[109246]: osdmap e201: 8 total, 7 up, 7 in 2024-08-17T17:27:36.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:36.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:36.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:36.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:36.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:36.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:37.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.005 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:37.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:37.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 101648 101632 76.5059 18 0.00406353 0.0129662 2024-08-17T17:27:37.091 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:36 smithi175 podman[143285]: 2024-08-17 17:27:36.679790128 +0000 UTC m=+0.018593245 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:27:37.092 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:36 smithi175 podman[143285]: 2024-08-17 17:27:36.804749059 +0000 UTC m=+0.143552207 container create 1f28e3e6ad61c2db038b8840853634d0e59af82004c9e372b57ae6537da007bb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T17:27:37.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:37.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:37.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:37.259 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:37.362 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:37.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:37.402 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:37 smithi175 podman[143285]: 2024-08-17 17:27:37.091019656 +0000 UTC m=+0.429822762 container init 1f28e3e6ad61c2db038b8840853634d0e59af82004c9e372b57ae6537da007bb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T17:27:37.402 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:37 smithi175 podman[143285]: 2024-08-17 17:27:37.095522722 +0000 UTC m=+0.434325828 container start 1f28e3e6ad61c2db038b8840853634d0e59af82004c9e372b57ae6537da007bb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:27:37.402 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:37 smithi175 podman[143285]: 2024-08-17 17:27:37.191122513 +0000 UTC m=+0.529925617 container attach 1f28e3e6ad61c2db038b8840853634d0e59af82004c9e372b57ae6537da007bb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-ced649f) 2024-08-17T17:27:37.402 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:27:37 smithi175 podman[143285]: 2024-08-17 17:27:37.305459856 +0000 UTC m=+0.644262974 container died 1f28e3e6ad61c2db038b8840853634d0e59af82004c9e372b57ae6537da007bb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:27:37.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:37.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:37.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:37.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:37 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:27:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:37 smithi175 ceph-mon[101434]: osdmap e202: 8 total, 7 up, 7 in 2024-08-17T17:27:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:37 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:27:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:37 smithi064 ceph-mon[104516]: osdmap e202: 8 total, 7 up, 7 in 2024-08-17T17:27:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:37 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:27:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:37 smithi064 ceph-mon[109246]: osdmap e202: 8 total, 7 up, 7 in 2024-08-17T17:27:37.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:37.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:37.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:37.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:38.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:38.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 102303 102287 76.0825 40.9375 0.0145097 0.0131376 2024-08-17T17:27:38.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:38.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:27:38.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:38.174 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:27:38.174 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:27:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:38.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:38.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:38.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:38.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:38.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:38.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:38.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:38.579 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:38.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[104516]: pgmap v600: 99 pgs: 7 active+undersized, 9 peering, 4 stale+active+clean, 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 71 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2202/301680 objects degraded (0.730%); 9636/301680 objects misplaced (3.194%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[104516]: 6.0 scrub ok 2024-08-17T17:27:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:27:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2202/301680 objects degraded (0.730%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:38.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[109246]: pgmap v600: 99 pgs: 7 active+undersized, 9 peering, 4 stale+active+clean, 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 71 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2202/301680 objects degraded (0.730%); 9636/301680 objects misplaced (3.194%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:38.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[109246]: 6.0 scrub ok 2024-08-17T17:27:38.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:27:38.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2202/301680 objects degraded (0.730%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:38.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:38.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:38.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:38 smithi175 ceph-mon[101434]: pgmap v600: 99 pgs: 7 active+undersized, 9 peering, 4 stale+active+clean, 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 71 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2202/301680 objects degraded (0.730%); 9636/301680 objects misplaced (3.194%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:27:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:38 smithi175 ceph-mon[101434]: 6.0 scrub ok 2024-08-17T17:27:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:38 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:27:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2202/301680 objects degraded (0.730%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:38.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.979 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:38.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 103207 103191 75.8521 56.5 0.00259675 0.0131781 2024-08-17T17:27:39.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.274 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:39.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:39.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:39.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:39.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:39.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:39.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:39.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:39.822 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:39.863 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:27:39.863 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:27:39.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:103980 6.13 6:c98fdb66:::benchmark_data_smithi064_185391_object103979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.836036+0000", 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.014512883000000001, 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014545338, 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:39.864 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 103980 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836036+0000", 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836036+0000", 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836037+0000", 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.598e-06 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836067+0000", 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9930999999999999e-05 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836086+0000", 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8746999999999999e-05 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836094+0000", 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6160000000000002e-06 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:39.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842283+0000", 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006188809 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842521+0000", 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000237502 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842699+0000", 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017783999999999999 2024-08-17T17:27:39.868 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.13 202/201 [PushReplyOp(6:c8f3281d:::benchmark_data_smithi064_185391_object5610:head)])", 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.836200+0000", 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.014348553999999999, 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014442847999999999, 2024-08-17T17:27:39.869 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836200+0000", 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836200+0000", 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836200+0000", 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.96e-07 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836204+0000", 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8190000000000002e-06 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836212+0000", 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9549999999999992e-06 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.836219+0000", 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4350000000000002e-06 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.872 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35007.0:103984 6.10 e202/201 6:0af62cc1:::benchmark_data_smithi064_185391_object103983:head v 202'4905, mlcod=177'3558)", 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.842521+0000", 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0080269790000000001, 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081610369999999995, 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842521+0000", 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842521+0000", 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842523+0000", 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.19e-06 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842547+0000", 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4036000000000002e-05 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842557+0000", 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0570999999999999e-05 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842563+0000", 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9030000000000001e-06 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.842713+0000", 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014971599999999999 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.846455+0000", 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037423909999999999 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:103985 6.f 6:f36f5f82:::benchmark_data_smithi064_185391_object103984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.843137+0000", 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0074114230000000003, 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007595185, 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 103985 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843137+0000", 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843137+0000", 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843139+0000", 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.53e-06 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843193+0000", 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4546000000000003e-05 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843205+0000", 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2016e-05 2024-08-17T17:27:39.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843212+0000", 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843390+0000", 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017765899999999999 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:39.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843510+0000", 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120745 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843629+0000", 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011849 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:103986 6.f 6.6eaacc3f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e202)", 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.843232+0000", 2024-08-17T17:27:39.881 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0073166799999999999, 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075701470000000002, 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 103986 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843232+0000", 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843232+0000", 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843232+0000", 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3000000000000001e-07 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843257+0000", 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4309000000000001e-05 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843264+0000", 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2089999999999999e-06 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.843269+0000", 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0030000000000002e-06 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35007.0:103987 6.18 e202/176)", 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.844098+0000", 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0064504180000000003, 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067417629999999996, 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.885 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844098+0000", 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844098+0000", 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844098+0000", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.39e-07 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844132+0000", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3433999999999999e-05 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844141+0000", 2024-08-17T17:27:39.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0739999999999999e-06 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.844147+0000", 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8769999999999998e-06 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35007.0:103985 6.f e202/167)", 2024-08-17T17:27:39.888 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.850521+0000", 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 2.7018000000000001e-05, 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000350014, 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850521+0000", 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850521+0000", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850522+0000", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7400000000000002e-07 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850526+0000", 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.641e-06 2024-08-17T17:27:39.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850539+0000", 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3378e-05 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.850547+0000", 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9019999999999999e-06 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35007.0:103975 6.3 e202/200 6:c1b8ab89:::benchmark_data_smithi064_185391_object103974:head v 202'4894, mlcod=202'4892)", 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:39.829068+0000", 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021480717, 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021848776, 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829068+0000", 2024-08-17T17:27:39.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829068+0000", 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829068+0000", 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0100000000000004e-07 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:27:39.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829125+0000", 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6585000000000001e-05 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829143+0000", 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7669999999999999e-05 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829149+0000", 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7530000000000004e-06 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829548+0000", 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039851300000000002 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:39.829568+0000", 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0069999999999999e-05 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:27:39.895 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2024-08-17T17:27:39.896 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:27:39.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:39.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:40.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 104157 104141 75.6605 59.375 0.0146757 0.0132116 2024-08-17T17:27:40.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:40.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:40.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:40.246 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:40.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:40.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:40.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:40.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:40.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:40.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[104516]: pgmap v602: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 peering, 1 stale+active+clean, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded, 63 active+clean; 6.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 17310/306648 objects degraded (5.645%); 12941/306648 objects misplaced (4.220%) 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[109246]: pgmap v602: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 peering, 1 stale+active+clean, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded, 63 active+clean; 6.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 17310/306648 objects degraded (5.645%); 12941/306648 objects misplaced (4.220%) 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:40.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:40.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:40.765 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:40.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:27:40.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:40 smithi175 ceph-mon[101434]: pgmap v602: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 peering, 1 stale+active+clean, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded, 63 active+clean; 6.2 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 17310/306648 objects degraded (5.645%); 12941/306648 objects misplaced (4.220%) 2024-08-17T17:27:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:40.921 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:40.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:41.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 105095 105079 75.4647 58.625 0.0138579 0.013245 2024-08-17T17:27:41.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:41.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:41.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:41.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:41.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:41.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:41.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:41.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:41.548 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:41.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:41.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:41.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:41.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:41.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:41.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:41.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:41.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:42.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 106047 106031 75.2832 59.5 0.00695727 0.0132781 2024-08-17T17:27:42.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:42.122 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:42.224 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:42.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:42.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:42.389 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:42.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:42.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:42.444 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:42.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:42.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.589 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:42.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:42.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:42 smithi064 ceph-mon[104516]: pgmap v603: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized, 9 peering, 10 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 25842/307860 objects degraded (8.394%); 10219/307860 objects misplaced (3.319%) 2024-08-17T17:27:42.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:42 smithi064 ceph-mon[109246]: pgmap v603: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized, 9 peering, 10 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 25842/307860 objects degraded (8.394%); 10219/307860 objects misplaced (3.319%) 2024-08-17T17:27:42.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:42.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:42.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:42.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:42 smithi175 ceph-mon[101434]: pgmap v603: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 20 active+undersized, 9 peering, 10 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 25842/307860 objects degraded (8.394%); 10219/307860 objects misplaced (3.319%) 2024-08-17T17:27:42.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:43.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:43.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 106955 106939 75.0749 56.75 0.0207246 0.0133139 2024-08-17T17:27:43.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:43.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:43.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:43.175 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:27:43.175 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:27:43.175 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:27:43.175 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2024-08-17T17:27:43.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T17:27:43.238 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:43.262 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:43.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:43.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:43.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:43.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25842/307860 objects degraded (8.394%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:43.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25842/307860 objects degraded (8.394%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:43.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:43.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:43.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:43.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:43.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:43.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:43.883 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:43.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25842/307860 objects degraded (8.394%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:43.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0107 2024-08-17T17:27:44.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 107730 2024-08-17T17:27:44.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:27:44.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:27:44.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 74.8036 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 44.4833 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 201.125 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1196 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 711.741 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3218 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0133633 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0627591 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.46805 2024-08-17T17:27:44.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00132797 2024-08-17T17:27:44.044 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:27:44.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:44.279 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:44.381 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.405 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:43] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:27:44.507 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:44.630 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:44.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:44.756 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:44.778 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:44.881 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:44.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:44.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:44 smithi175 ceph-mon[101434]: pgmap v604: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35722/311004 objects degraded (11.486%); 10219/311004 objects misplaced (3.286%) 2024-08-17T17:27:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:44 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:27:44.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:44 smithi064 ceph-mon[104516]: pgmap v604: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35722/311004 objects degraded (11.486%); 10219/311004 objects misplaced (3.286%) 2024-08-17T17:27:44.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:44 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:27:44.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:44 smithi064 ceph-mon[109246]: pgmap v604: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 35722/311004 objects degraded (11.486%); 10219/311004 objects misplaced (3.286%) 2024-08-17T17:27:44.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:44 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:27:45.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:45.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:45.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.195 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:45.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:45.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:45.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:45.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:45.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:45.562 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:45.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:45.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:45.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:45.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2559015061' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2559015061' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2559015061' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:45.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:45.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:46.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:46.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:46.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:46.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:46.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:46.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:46.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:46.542 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:46.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:46.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:46.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:46.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:46 smithi175 ceph-mon[101434]: pgmap v605: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 116 B/s rd, 0 op/s; 36545/320775 objects degraded (11.393%); 9769/320775 objects misplaced (3.045%) 2024-08-17T17:27:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:46 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:46 smithi175 ceph-mon[101434]: osdmap e203: 8 total, 7 up, 7 in 2024-08-17T17:27:46.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[104516]: pgmap v605: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 116 B/s rd, 0 op/s; 36545/320775 objects degraded (11.393%); 9769/320775 objects misplaced (3.045%) 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[104516]: osdmap e203: 8 total, 7 up, 7 in 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[109246]: pgmap v605: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 116 B/s rd, 0 op/s; 36545/320775 objects degraded (11.393%); 9769/320775 objects misplaced (3.045%) 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:46 smithi064 ceph-mon[109246]: osdmap e203: 8 total, 7 up, 7 in 2024-08-17T17:27:46.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:47.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:47.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:47.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:47.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:47.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:47.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:47.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:47.544 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:47.630 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:27:47.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:47.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:47.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:47.813 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:47.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:47.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:48.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:48.076 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:48.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:48.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:48.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:48.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:48.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:48.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:48.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:48.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:48 smithi064 ceph-mon[104516]: pgmap v607: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 0 op/s; 36656/321432 objects degraded (11.404%); 9769/321432 objects misplaced (3.039%) 2024-08-17T17:27:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 36656/321432 objects degraded (11.404%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:48 smithi064 ceph-mon[109246]: pgmap v607: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 0 op/s; 36656/321432 objects degraded (11.404%); 9769/321432 objects misplaced (3.039%) 2024-08-17T17:27:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 36656/321432 objects degraded (11.404%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:48.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:48.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:48.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:48.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:48 smithi175 ceph-mon[101434]: pgmap v607: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 0 op/s; 36656/321432 objects degraded (11.404%); 9769/321432 objects misplaced (3.039%) 2024-08-17T17:27:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 36656/321432 objects degraded (11.404%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:48.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:49.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:49.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:49.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:49.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:49.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:49.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:49.489 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:49.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:49.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:49.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:49.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:49.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:49.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:49.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:49.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:50.012 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:50.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.157 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:50.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:50.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.304 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:50.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:50.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:50.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:50.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:50 smithi064 ceph-mon[104516]: pgmap v608: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 853 op/s; 36047/314697 objects degraded (11.455%); 9389/314697 objects misplaced (2.984%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:27:50.744 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:50 smithi064 ceph-mon[109246]: pgmap v608: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 853 op/s; 36047/314697 objects degraded (11.455%); 9389/314697 objects misplaced (2.984%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:27:50.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:50.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:50.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:50 smithi175 ceph-mon[101434]: pgmap v608: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 853 op/s; 36047/314697 objects degraded (11.455%); 9389/314697 objects misplaced (2.984%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:27:50.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:50.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:51.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:51.079 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:27:51.079 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:27:51.079 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:27:51.079 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:27:51.079 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:27:51.116 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:51.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:51.241 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:51.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:51.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:51.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:51.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:51.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:51.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:51.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:51.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:51.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:52.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:52.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:52.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:52.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:52.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:52.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:52.390 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:52.471 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:27:52.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:52.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:52.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:52.505 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:27:52.506 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:52.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:27:52.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:52 smithi064 ceph-mon[109246]: pgmap v609: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 890 op/s; 35590/312366 objects degraded (11.394%); 9077/312366 objects misplaced (2.906%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:52.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1734743494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:52.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:52 smithi064 ceph-mon[104516]: pgmap v609: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 890 op/s; 35590/312366 objects degraded (11.394%); 9077/312366 objects misplaced (2.906%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:52.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1734743494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:52.540 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:27:52.540 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:52.583 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:27:52.584 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:52.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:52.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:27:52.619 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:52.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:52.664 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:27:52.664 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:52.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:27:52.719 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:27:52.719 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:52.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:52.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:52.899 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:52.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:52 smithi175 ceph-mon[101434]: pgmap v609: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 890 op/s; 35590/312366 objects degraded (11.394%); 9077/312366 objects misplaced (2.906%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1734743494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:52.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:53.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:53.046 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:53.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:53.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:27:53.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:53.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:53.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:53.316 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:53.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:53.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:53.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:53.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:53.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 35590/312366 objects degraded (11.394%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 35590/312366 objects degraded (11.394%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:53.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:53.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:53.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 35590/312366 objects degraded (11.394%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:53.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:27:53.998 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:27:54.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:54.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:54.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:54.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:54.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.426 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:54.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.439 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:27:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:27:53] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:27:54.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:54.590 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:54.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:54.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:54.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:54.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:54.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:54 smithi175 ceph-mon[101434]: pgmap v610: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 921 op/s; 35162/308307 objects degraded (11.405%); 9077/308307 objects misplaced (2.944%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:54 smithi064 ceph-mon[104516]: pgmap v610: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 921 op/s; 35162/308307 objects degraded (11.405%); 9077/308307 objects misplaced (2.944%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:54 smithi064 ceph-mon[109246]: pgmap v610: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 921 op/s; 35162/308307 objects degraded (11.405%); 9077/308307 objects misplaced (2.944%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:27:54.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:55.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:55.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:55.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:55.235 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:55.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:55.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:55.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:55.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:55.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:55.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:55.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:55.790 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:55.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:55.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:55.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:56.055 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:56.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:56.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:56.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:56.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:56.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:56.329 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:56 smithi175 ceph-mon[101434]: pgmap v611: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.5 MiB/s wr, 941 op/s; 33937/297765 objects degraded (11.397%); 8240/297765 objects misplaced (2.767%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:27:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:56.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[109246]: pgmap v611: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.5 MiB/s wr, 941 op/s; 33937/297765 objects degraded (11.397%); 8240/297765 objects misplaced (2.767%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:27:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[104516]: pgmap v611: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.5 MiB/s wr, 941 op/s; 33937/297765 objects degraded (11.397%); 8240/297765 objects misplaced (2.767%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:27:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:27:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:56.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:56.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:56.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:56.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:56.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:56.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:56.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.883 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:56.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:56.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:56.999 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:57.008 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.232 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:57.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.362 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:57.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:57.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:57.662 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:57.764 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:57.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:57.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.914 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.016 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:58.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:58 smithi175 ceph-mon[101434]: pgmap v612: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.8 MiB/s wr, 937 op/s; 33500/293829 objects degraded (11.401%); 8240/293829 objects misplaced (2.804%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:27:58.165 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:58.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:58.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:58.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:58 smithi064 ceph-mon[104516]: pgmap v612: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.8 MiB/s wr, 937 op/s; 33500/293829 objects degraded (11.401%); 8240/293829 objects misplaced (2.804%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:27:58.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:58 smithi064 ceph-mon[109246]: pgmap v612: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 4.8 MiB/s wr, 937 op/s; 33500/293829 objects degraded (11.401%); 8240/293829 objects misplaced (2.804%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:27:58.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.434 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:58.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:58.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:58.557 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:58.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:58.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:58.802 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.926 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:58.948 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:59.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:59.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:59.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:59.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:59.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:59.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:59.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:27:59 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 33500/293829 objects degraded (11.401%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:27:59 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 33500/293829 objects degraded (11.401%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:27:59 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 33500/293829 objects degraded (11.401%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:27:59.470 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:59.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:59.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:59.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:59.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:59.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:27:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:59.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:59.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:27:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:00.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.026 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:00.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:00 smithi064 ceph-mon[104516]: pgmap v613: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 810 KiB/s wr, 1.01k op/s; 32737/285810 objects degraded (11.454%); 7778/285810 objects misplaced (2.721%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:00 smithi064 ceph-mon[109246]: pgmap v613: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 810 KiB/s wr, 1.01k op/s; 32737/285810 objects degraded (11.454%); 7778/285810 objects misplaced (2.721%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:00.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:00.305 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:00.407 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:00 smithi175 ceph-mon[101434]: pgmap v613: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 810 KiB/s wr, 1.01k op/s; 32737/285810 objects degraded (11.454%); 7778/285810 objects misplaced (2.721%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:00.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:00.571 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:00.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:00.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:00.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:00.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:01.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:01.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:01.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:01.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:01.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:01.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4059472240' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:01 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4059472240' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:01 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4059472240' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:01 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:01.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:01.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:01.627 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:01.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:01.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:01.774 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:01.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:01.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:02.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:02.026 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:02.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:02.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:02.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[104516]: pgmap v614: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 32284/283743 objects degraded (11.378%); 7438/283743 objects misplaced (2.621%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:02.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:02.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[104516]: osdmap e204: 8 total, 7 up, 7 in 2024-08-17T17:28:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[109246]: pgmap v614: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 32284/283743 objects degraded (11.378%); 7438/283743 objects misplaced (2.621%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:02.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:02 smithi064 ceph-mon[109246]: osdmap e204: 8 total, 7 up, 7 in 2024-08-17T17:28:02.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:02.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:02.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:02.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:02 smithi175 ceph-mon[101434]: pgmap v614: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 32284/283743 objects degraded (11.378%); 7438/283743 objects misplaced (2.621%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:02 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:02 smithi175 ceph-mon[101434]: osdmap e204: 8 total, 7 up, 7 in 2024-08-17T17:28:02.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:02.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:02.521 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:02.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:02.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:02.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:02.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:02.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:02.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:02.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:02.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:02.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:03.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:03.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:03.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:03.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:03.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:03.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:03.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:03.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:03.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:03.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:03.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:03.813 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:03.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:03.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:03.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:04.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:04.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:04.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:04.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:04.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:04.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:04 smithi175 ceph-mon[101434]: pgmap v616: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 31886/279945 objects degraded (11.390%); 7438/279945 objects misplaced (2.657%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:04 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 31886/279945 objects degraded (11.390%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:28:04.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:03] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-08-17T17:28:04.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:04 smithi064 ceph-mon[104516]: pgmap v616: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 31886/279945 objects degraded (11.390%); 7438/279945 objects misplaced (2.657%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:04.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:04 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 31886/279945 objects degraded (11.390%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:28:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:04 smithi064 ceph-mon[109246]: pgmap v616: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 31886/279945 objects degraded (11.390%); 7438/279945 objects misplaced (2.657%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:04 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 31886/279945 objects degraded (11.390%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T17:28:04.473 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:04.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:04.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:04.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:04.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:04.768 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:04.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:04.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:04.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:05.017 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:05.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:05.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:05.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:05.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:05.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.512 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:05.534 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:05.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:05.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:05.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.824 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:05.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:05.925 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:05.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:06.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:06.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:06.187 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:06.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:06.247 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:06.247 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:06.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:06 smithi175 ceph-mon[101434]: pgmap v617: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30643/269712 objects degraded (11.361%); 6604/269712 objects misplaced (2.449%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:06 smithi064 ceph-mon[104516]: pgmap v617: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30643/269712 objects degraded (11.361%); 6604/269712 objects misplaced (2.449%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:06 smithi064 ceph-mon[109246]: pgmap v617: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30643/269712 objects degraded (11.361%); 6604/269712 objects misplaced (2.449%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:06.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:06.499 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:06.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:06.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:06.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:06.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:06.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:06.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:06.945 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:28:06.945 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:28:06.945 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-08-17T17:28:06.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:28:07.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:07.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:07.165 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:07.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:07.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:07.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:07.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:07.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:07.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:07.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:07.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.728 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:07.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:07.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:07.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:07.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:08.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:08.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:08.126 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:08.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:08 smithi175 ceph-mon[101434]: pgmap v618: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30206/265770 objects degraded (11.365%); 6604/265770 objects misplaced (2.485%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:08.227 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:08.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:08.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:08.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:08.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:08 smithi064 ceph-mon[104516]: pgmap v618: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30206/265770 objects degraded (11.365%); 6604/265770 objects misplaced (2.485%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:08.429 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:08 smithi064 ceph-mon[109246]: pgmap v618: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 30206/265770 objects degraded (11.365%); 6604/265770 objects misplaced (2.485%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:08.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:08.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:08.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:08.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:08.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:08.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:08.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:08.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:08.914 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:09.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:09.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:09.176 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:09.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:28:09.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:09 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 30206/265770 objects degraded (11.365%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:09.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:09 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 30206/265770 objects degraded (11.365%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:09.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:09 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 30206/265770 objects degraded (11.365%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:09.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:09.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.670 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:09.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:09.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:10.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:10.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:10.161 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[104516]: pgmap v619: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 29427/257610 objects degraded (11.423%); 6137/257610 objects misplaced (2.382%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1646271276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[109246]: pgmap v619: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 29427/257610 objects degraded (11.423%); 6137/257610 objects misplaced (2.382%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:10.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1646271276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:10.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:10.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:10.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:10.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:10 smithi175 ceph-mon[101434]: pgmap v619: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 29427/257610 objects degraded (11.423%); 6137/257610 objects misplaced (2.382%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:28:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1646271276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:10.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:10.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:10.542 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:10.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:10.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:10.768 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:10.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:10.895 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:10.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:10.983 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:11.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:11.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:11.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:11.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:11.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:11.323 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:11.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:11.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[104516]: osdmap e205: 8 total, 7 up, 7 in 2024-08-17T17:28:11.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:11.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:11.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:11 smithi064 ceph-mon[109246]: osdmap e205: 8 total, 7 up, 7 in 2024-08-17T17:28:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.357 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:11.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:11 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:11 smithi175 ceph-mon[101434]: osdmap e205: 8 total, 7 up, 7 in 2024-08-17T17:28:11.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:11.496 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:11.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:11.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:11.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:11.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:11.746 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:11.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:11.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:11.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:11.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:12.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:12.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:12.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:12.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:12.270 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:12.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:12.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:12 smithi175 ceph-mon[101434]: pgmap v621: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 28927/255480 objects degraded (11.323%); 5741/255480 objects misplaced (2.247%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:28:12.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:12.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:12 smithi064 ceph-mon[104516]: pgmap v621: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 28927/255480 objects degraded (11.323%); 5741/255480 objects misplaced (2.247%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:28:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:12 smithi064 ceph-mon[109246]: pgmap v621: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 28927/255480 objects degraded (11.323%); 5741/255480 objects misplaced (2.247%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:28:12.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:12.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:12.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:12.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:12.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:12.701 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:12.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:12.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:12.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:12.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:13.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:28:13.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:13.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:13.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:13.182 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:13.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:13.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28927/255480 objects degraded (11.323%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:13.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28927/255480 objects degraded (11.323%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:13.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:13.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:13.654 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:13.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28927/255480 objects degraded (11.323%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:13.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:28:13.680 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:13.714 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:28:13.714 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:13.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:28:13.758 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:13.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:13.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:13.809 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:28:13.809 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:13.844 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:28:13.844 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:13.878 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:28:13.878 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:13.879 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:28:13.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:13.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:28:13.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:13.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:14.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:14.101 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:13] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:28:14.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:14.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:14.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:14.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:14 smithi064 ceph-mon[104516]: pgmap v622: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 28483/251562 objects degraded (11.322%); 5741/251562 objects misplaced (2.282%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:28:14.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1753188885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:14.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:14 smithi064 ceph-mon[109246]: pgmap v622: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 28483/251562 objects degraded (11.322%); 5741/251562 objects misplaced (2.282%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:28:14.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1753188885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:14.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:14.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:14.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:14 smithi175 ceph-mon[101434]: pgmap v622: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 28483/251562 objects degraded (11.322%); 5741/251562 objects misplaced (2.282%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:28:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1753188885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:14.667 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:14.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:14.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:14.828 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:14.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:14.930 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:14.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:15.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:15.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:15.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:15.120 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:15.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:15.249 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:15.305 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:28:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:28:15.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:15.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:15.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:15.532 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:15.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:15.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:15.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:15.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:15.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:15.927 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:16.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:16.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:16.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:16.193 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:16.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:16.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:16.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:16.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:16 smithi064 ceph-mon[104516]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 27298/241698 objects degraded (11.294%); 4832/241698 objects misplaced (1.999%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:16.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:16 smithi064 ceph-mon[104516]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:28:16.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:16 smithi064 ceph-mon[109246]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 27298/241698 objects degraded (11.294%); 4832/241698 objects misplaced (1.999%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:16.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:16 smithi064 ceph-mon[109246]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:28:16.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:16.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:16.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:16 smithi175 ceph-mon[101434]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 27298/241698 objects degraded (11.294%); 4832/241698 objects misplaced (1.999%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:16 smithi175 ceph-mon[101434]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:28:16.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:16.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:16.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:16.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:16.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:17.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:17.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:17.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:17.308 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:17.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:17.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:17.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:17.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:17.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:17.823 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:17.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:17.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:17.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:18.096 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:18.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:18.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:18.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:18.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:18.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:18 smithi175 ceph-mon[101434]: pgmap v624: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 26841/237894 objects degraded (11.283%); 4832/237894 objects misplaced (2.031%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 26841/237894 objects degraded (11.283%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:18 smithi064 ceph-mon[104516]: pgmap v624: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 26841/237894 objects degraded (11.283%); 4832/237894 objects misplaced (2.031%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 26841/237894 objects degraded (11.283%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:18 smithi064 ceph-mon[109246]: pgmap v624: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 26841/237894 objects degraded (11.283%); 4832/237894 objects misplaced (2.031%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:28:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 26841/237894 objects degraded (11.283%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:18.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:18.761 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.863 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:18.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:18.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:18.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.012 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:19.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.139 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:19.283 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:19.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:19.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:19.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:19.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:19.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:19.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:19.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:19.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:19.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:19.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:19.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.992 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.094 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.116 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:20.242 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:20.300 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":623,"stamp":"2024-08-17T17:28:14.931004+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5282863101,"num_objects":80566,"num_object_clones":0,"num_object_copies":241698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27298,"num_objects_misplaced":4832,"num_objects_unfound":0,"num_objects_dirty":80566,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":117284,"num_write_kb":5704514,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9120,"num_bytes_recovered":601496759,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":159066,"ondisk_log_size":159066,"up":256,"acting":256,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19150332,"kb_used_data":17058748,"kb_used_omap":33,"kb_used_meta":2091422,"kb_avail":637065732,"statfs":{"total":671965249536,"available":652355309568,"internally_reserved":0,"allocated":17468157952,"data_stored":17460488254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34229,"internal_metadata":2141616715},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-611975168,"num_objects":-9338,"num_object_clones":0,"num_object_copies":-28014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3345,"num_objects_misplaced":-1772,"num_objects_unfound":0,"num_objects_dirty":-9338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9338,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":736,"num_bytes_recovered":48234496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004796"},"pg_stats":[{"pgid":"6.16","version":"205'6046","reported_seq":6170,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.863595+0000","last_change":"2024-08-17T17:28:08.917625+0000","last_active":"2024-08-17T17:28:13.863595+0000","last_peered":"2024-08-17T17:28:13.863595+0000","last_clean":"2024-08-17T17:28:13.863595+0000","last_became_active":"2024-08-17T17:26:42.142775+0000","last_became_peered":"2024-08-17T17:26:42.142775+0000","last_unstale":"2024-08-17T17:28:13.863595+0000","last_undegraded":"2024-08-17T17:28:13.863595+0000","last_fullsized":"2024-08-17T17:28:13.863595+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164233216,"num_objects":2506,"num_object_clones":0,"num_object_copies":7518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4350,"num_write_kb":219392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":266,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.162641+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:28:10.162641+0000","last_peered":"2024-08-17T17:28:10.162641+0000","last_clean":"2024-08-17T17:28:10.162641+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:28:10.162641+0000","last_undegraded":"2024-08-17T17:28:10.162641+0000","last_fullsized":"2024-08-17T17:28:10.162641+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":200,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.107422+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:28:01.107422+0000","last_peered":"2024-08-17T17:28:01.107422+0000","last_clean":"2024-08-17T17:28:01.107422+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:28:01.107422+0000","last_undegraded":"2024-08-17T17:28:01.107422+0000","last_fullsized":"2024-08-17T17:28:01.107422+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"205'5942","reported_seq":6070,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.902955+0000","last_change":"2024-08-17T17:28:10.922238+0000","last_active":"2024-08-17T17:28:13.902955+0000","last_peered":"2024-08-17T17:28:13.902955+0000","last_clean":"2024-08-17T17:28:13.902955+0000","last_became_active":"2024-08-17T17:26:42.150796+0000","last_became_peered":"2024-08-17T17:26:42.150796+0000","last_unstale":"2024-08-17T17:28:13.902955+0000","last_undegraded":"2024-08-17T17:28:13.902955+0000","last_fullsized":"2024-08-17T17:28:13.902955+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:28:15.892005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159154176,"num_objects":2429,"num_object_clones":0,"num_object_copies":7287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4251,"num_write_kb":213728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.107173+0000","last_change":"2024-08-17T17:27:36.164133+0000","last_active":"2024-08-17T17:28:01.107173+0000","last_peered":"2024-08-17T17:28:01.107173+0000","last_clean":"2024-08-17T17:27:34.151200+0000","last_became_active":"2024-08-17T17:27:36.164133+0000","last_became_peered":"2024-08-17T17:27:36.164133+0000","last_unstale":"2024-08-17T17:28:01.107173+0000","last_undegraded":"2024-08-17T17:28:01.107173+0000","last_fullsized":"2024-08-17T17:27:36.152969+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:30:45.809544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176864+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:28:10.176864+0000","last_peered":"2024-08-17T17:28:10.176864+0000","last_clean":"2024-08-17T17:28:10.176864+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:28:10.176864+0000","last_undegraded":"2024-08-17T17:28:10.176864+0000","last_fullsized":"2024-08-17T17:28:10.176864+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"205'6014","reported_seq":12042,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:28:14.330248+0000","last_change":"2024-08-17T17:27:35.171409+0000","last_active":"2024-08-17T17:28:14.330248+0000","last_peered":"2024-08-17T17:28:14.330248+0000","last_clean":"2024-08-17T17:27:01.214262+0000","last_became_active":"2024-08-17T17:27:03.256580+0000","last_became_peered":"2024-08-17T17:27:03.256580+0000","last_unstale":"2024-08-17T17:28:14.330248+0000","last_undegraded":"2024-08-17T17:27:35.170737+0000","last_fullsized":"2024-08-17T17:27:03.147067+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":6014,"log_dups_size":0,"ondisk_log_size":6014,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165216256,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":820,"num_objects_misplaced":1640,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4313,"num_write_kb":218688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1480,"num_bytes_recovered":96993280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":1701},{"shards":"1,3","objects":820}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":257,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.174365+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:28:10.174365+0000","last_peered":"2024-08-17T17:28:10.174365+0000","last_clean":"2024-08-17T17:28:10.174365+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:28:10.174365+0000","last_undegraded":"2024-08-17T17:28:10.174365+0000","last_fullsized":"2024-08-17T17:28:10.174365+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":177,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.107623+0000","last_change":"2024-08-17T17:27:36.155814+0000","last_active":"2024-08-17T17:28:01.107623+0000","last_peered":"2024-08-17T17:28:01.107623+0000","last_clean":"2024-08-17T17:27:34.150951+0000","last_became_active":"2024-08-17T17:27:36.155814+0000","last_became_peered":"2024-08-17T17:27:36.155814+0000","last_unstale":"2024-08-17T17:28:01.107623+0000","last_undegraded":"2024-08-17T17:28:01.107623+0000","last_fullsized":"2024-08-17T17:27:36.151421+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:37.003689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"205'5799","reported_seq":10359,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:11.002119+0000","last_change":"2024-08-17T17:27:36.154870+0000","last_active":"2024-08-17T17:28:11.002119+0000","last_peered":"2024-08-17T17:28:11.002119+0000","last_clean":"2024-08-17T17:27:34.939677+0000","last_became_active":"2024-08-17T17:27:36.154870+0000","last_became_peered":"2024-08-17T17:27:36.154870+0000","last_unstale":"2024-08-17T17:28:11.002119+0000","last_undegraded":"2024-08-17T17:27:36.151574+0000","last_fullsized":"2024-08-17T17:27:36.151470+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169672704,"num_objects":2589,"num_object_clones":0,"num_object_copies":7767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4115,"num_write_kb":214528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2239,"num_bytes_recovered":146735104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2589}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":265,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.160912+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:28:10.160912+0000","last_peered":"2024-08-17T17:28:10.160912+0000","last_clean":"2024-08-17T17:28:10.160912+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:28:10.160912+0000","last_undegraded":"2024-08-17T17:28:10.160912+0000","last_fullsized":"2024-08-17T17:28:10.160912+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":29,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:10.169044+0000","last_change":"2024-08-17T17:27:36.168172+0000","last_active":"2024-08-17T17:28:10.169044+0000","last_peered":"2024-08-17T17:28:10.169044+0000","last_clean":"2024-08-17T17:05:36.188280+0000","last_became_active":"2024-08-17T17:27:36.168172+0000","last_became_peered":"2024-08-17T17:27:36.168172+0000","last_unstale":"2024-08-17T17:28:10.169044+0000","last_undegraded":"2024-08-17T17:27:36.153125+0000","last_fullsized":"2024-08-17T17:27:36.153020+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:59.647578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"205'8513","reported_seq":8570,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:13.570195+0000","last_change":"2024-08-17T17:27:36.160640+0000","last_active":"2024-08-17T17:28:13.570195+0000","last_peered":"2024-08-17T17:28:13.570195+0000","last_clean":"2024-08-17T17:27:34.929974+0000","last_became_active":"2024-08-17T17:27:36.160640+0000","last_became_peered":"2024-08-17T17:27:36.160640+0000","last_unstale":"2024-08-17T17:28:13.570195+0000","last_undegraded":"2024-08-17T17:27:36.154223+0000","last_fullsized":"2024-08-17T17:27:36.154111+0000","mapping_epoch":200,"log_start":"171'800","ondisk_log_start":"171'800","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":7713,"log_dups_size":0,"ondisk_log_size":7713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:06.129495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4901,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8513,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4901}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.107045+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:28:01.107045+0000","last_peered":"2024-08-17T17:28:01.107045+0000","last_clean":"2024-08-17T17:28:01.107045+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:28:01.107045+0000","last_undegraded":"2024-08-17T17:28:01.107045+0000","last_fullsized":"2024-08-17T17:28:01.107045+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.106533+0000","last_change":"2024-08-17T17:27:36.154451+0000","last_active":"2024-08-17T17:28:01.106533+0000","last_peered":"2024-08-17T17:28:01.106533+0000","last_clean":"2024-08-17T17:27:34.150985+0000","last_became_active":"2024-08-17T17:27:36.154451+0000","last_became_peered":"2024-08-17T17:27:36.154451+0000","last_unstale":"2024-08-17T17:28:01.106533+0000","last_undegraded":"2024-08-17T17:28:01.106533+0000","last_fullsized":"2024-08-17T17:27:36.151416+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:45:31.211262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"205'8404","reported_seq":8500,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.880949+0000","last_change":"2024-08-17T17:26:02.539050+0000","last_active":"2024-08-17T17:28:13.880949+0000","last_peered":"2024-08-17T17:28:13.880949+0000","last_clean":"2024-08-17T17:28:13.880949+0000","last_became_active":"2024-08-17T17:26:02.538574+0000","last_became_peered":"2024-08-17T17:26:02.538574+0000","last_unstale":"2024-08-17T17:28:13.880949+0000","last_undegraded":"2024-08-17T17:28:13.880949+0000","last_fullsized":"2024-08-17T17:28:13.880949+0000","mapping_epoch":167,"log_start":"174'2800","ondisk_log_start":"174'2800","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:25:45.382148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325320704,"num_objects":4964,"num_object_clones":0,"num_object_copies":14892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8404,"num_write_kb":427776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:06.024395+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:28:06.024395+0000","last_peered":"2024-08-17T17:28:06.024395+0000","last_clean":"2024-08-17T17:28:06.024395+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:28:06.024395+0000","last_undegraded":"2024-08-17T17:28:06.024395+0000","last_fullsized":"2024-08-17T17:28:06.024395+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.116069+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:28:01.116069+0000","last_peered":"2024-08-17T17:28:01.116069+0000","last_clean":"2024-08-17T17:28:01.116069+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:28:01.116069+0000","last_undegraded":"2024-08-17T17:28:01.116069+0000","last_fullsized":"2024-08-17T17:28:01.116069+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"205'5780","reported_seq":5837,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.951711+0000","last_change":"2024-08-17T17:26:42.115209+0000","last_active":"2024-08-17T17:28:10.951711+0000","last_peered":"2024-08-17T17:28:10.951711+0000","last_clean":"2024-08-17T17:28:10.951711+0000","last_became_active":"2024-08-17T17:26:42.113429+0000","last_became_peered":"2024-08-17T17:26:42.113429+0000","last_unstale":"2024-08-17T17:28:10.951711+0000","last_undegraded":"2024-08-17T17:28:10.951711+0000","last_fullsized":"2024-08-17T17:28:10.951711+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166232064,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4104,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":174,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:06.024341+0000","last_change":"2024-08-17T17:27:36.164443+0000","last_active":"2024-08-17T17:28:06.024341+0000","last_peered":"2024-08-17T17:28:06.024341+0000","last_clean":"2024-08-17T17:27:34.151447+0000","last_became_active":"2024-08-17T17:27:36.164443+0000","last_became_peered":"2024-08-17T17:27:36.164443+0000","last_unstale":"2024-08-17T17:28:06.024341+0000","last_undegraded":"2024-08-17T17:27:36.153461+0000","last_fullsized":"2024-08-17T17:27:36.153358+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:23.781560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":154,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.165063+0000","last_change":"2024-08-17T17:27:36.157622+0000","last_active":"2024-08-17T17:28:10.165063+0000","last_peered":"2024-08-17T17:28:10.165063+0000","last_clean":"2024-08-17T17:26:20.952601+0000","last_became_active":"2024-08-17T17:27:36.157622+0000","last_became_peered":"2024-08-17T17:27:36.157622+0000","last_unstale":"2024-08-17T17:28:10.165063+0000","last_undegraded":"2024-08-17T17:28:10.165063+0000","last_fullsized":"2024-08-17T17:27:36.152425+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:18.183484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.9","version":"205'5867","reported_seq":5993,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.586025+0000","last_change":"2024-08-17T17:28:10.396216+0000","last_active":"2024-08-17T17:28:13.586025+0000","last_peered":"2024-08-17T17:28:13.586025+0000","last_clean":"2024-08-17T17:28:13.586025+0000","last_became_active":"2024-08-17T17:26:42.111639+0000","last_became_peered":"2024-08-17T17:26:42.111639+0000","last_unstale":"2024-08-17T17:28:13.586025+0000","last_undegraded":"2024-08-17T17:28:13.586025+0000","last_fullsized":"2024-08-17T17:28:13.586025+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:28:15.358272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167378944,"num_objects":2554,"num_object_clones":0,"num_object_copies":7662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4208,"num_write_kb":216384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":219,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.162802+0000","last_change":"2024-08-17T17:27:36.174335+0000","last_active":"2024-08-17T17:28:10.162802+0000","last_peered":"2024-08-17T17:28:10.162802+0000","last_clean":"2024-08-17T17:27:34.154090+0000","last_became_active":"2024-08-17T17:27:36.174335+0000","last_became_peered":"2024-08-17T17:27:36.174335+0000","last_unstale":"2024-08-17T17:28:10.162802+0000","last_undegraded":"2024-08-17T17:28:10.162802+0000","last_fullsized":"2024-08-17T17:27:36.155587+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:21.476040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":271,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176435+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:28:10.176435+0000","last_peered":"2024-08-17T17:28:10.176435+0000","last_clean":"2024-08-17T17:28:10.176435+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:28:10.176435+0000","last_undegraded":"2024-08-17T17:28:10.176435+0000","last_fullsized":"2024-08-17T17:28:10.176435+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"205'8352","reported_seq":8399,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:11.537341+0000","last_change":"2024-08-17T17:26:02.533130+0000","last_active":"2024-08-17T17:28:11.537341+0000","last_peered":"2024-08-17T17:28:11.537341+0000","last_clean":"2024-08-17T17:28:11.537341+0000","last_became_active":"2024-08-17T17:26:02.532783+0000","last_became_peered":"2024-08-17T17:26:02.532783+0000","last_unstale":"2024-08-17T17:28:11.537341+0000","last_undegraded":"2024-08-17T17:28:11.537341+0000","last_fullsized":"2024-08-17T17:28:11.537341+0000","mapping_epoch":167,"log_start":"172'1500","ondisk_log_start":"172'1500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":6852,"log_dups_size":0,"ondisk_log_size":6852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:32.674058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8352,"num_write_kb":431168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":287,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.160031+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:28:10.160031+0000","last_peered":"2024-08-17T17:28:10.160031+0000","last_clean":"2024-08-17T17:28:10.160031+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:28:10.160031+0000","last_undegraded":"2024-08-17T17:28:10.160031+0000","last_fullsized":"2024-08-17T17:28:10.160031+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":96,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.118947+0000","last_change":"2024-08-17T17:27:36.162164+0000","last_active":"2024-08-17T17:28:01.118947+0000","last_peered":"2024-08-17T17:28:01.118947+0000","last_clean":"2024-08-17T17:27:34.153084+0000","last_became_active":"2024-08-17T17:27:36.162164+0000","last_became_peered":"2024-08-17T17:27:36.162164+0000","last_unstale":"2024-08-17T17:28:01.118947+0000","last_undegraded":"2024-08-17T17:28:01.118947+0000","last_fullsized":"2024-08-17T17:27:36.150716+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:39.809556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"205'8609","reported_seq":8705,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.245271+0000","last_change":"2024-08-17T17:26:02.537749+0000","last_active":"2024-08-17T17:28:13.245271+0000","last_peered":"2024-08-17T17:28:13.245271+0000","last_clean":"2024-08-17T17:28:13.245271+0000","last_became_active":"2024-08-17T17:26:02.537124+0000","last_became_peered":"2024-08-17T17:26:02.537124+0000","last_unstale":"2024-08-17T17:28:13.245271+0000","last_undegraded":"2024-08-17T17:28:13.245271+0000","last_fullsized":"2024-08-17T17:28:13.245271+0000","mapping_epoch":167,"log_start":"171'700","ondisk_log_start":"171'700","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":7909,"log_dups_size":0,"ondisk_log_size":7909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:46.989948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333512704,"num_objects":5089,"num_object_clones":0,"num_object_copies":15267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8609,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.116481+0000","last_change":"2024-08-17T17:27:36.173059+0000","last_active":"2024-08-17T17:28:01.116481+0000","last_peered":"2024-08-17T17:28:01.116481+0000","last_clean":"2024-08-17T17:22:13.178082+0000","last_became_active":"2024-08-17T17:27:36.173059+0000","last_became_peered":"2024-08-17T17:27:36.173059+0000","last_unstale":"2024-08-17T17:28:01.116481+0000","last_undegraded":"2024-08-17T17:28:01.116481+0000","last_fullsized":"2024-08-17T17:27:36.150235+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:34.696702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.116684+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:28:01.116684+0000","last_peered":"2024-08-17T17:28:01.116684+0000","last_clean":"2024-08-17T17:28:01.116684+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:28:01.116684+0000","last_undegraded":"2024-08-17T17:28:01.116684+0000","last_fullsized":"2024-08-17T17:28:01.116684+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"205'8553","reported_seq":8652,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:13.920820+0000","last_change":"2024-08-17T17:27:36.161695+0000","last_active":"2024-08-17T17:28:13.920820+0000","last_peered":"2024-08-17T17:28:13.920820+0000","last_clean":"2024-08-17T17:27:34.916313+0000","last_became_active":"2024-08-17T17:27:36.161695+0000","last_became_peered":"2024-08-17T17:27:36.161695+0000","last_unstale":"2024-08-17T17:28:13.920820+0000","last_undegraded":"2024-08-17T17:27:36.155023+0000","last_fullsized":"2024-08-17T17:27:36.154914+0000","mapping_epoch":200,"log_start":"174'2900","ondisk_log_start":"174'2900","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5653,"log_dups_size":0,"ondisk_log_size":5653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:01:22.555393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330104832,"num_objects":5037,"num_object_clones":0,"num_object_copies":15111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5037,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8553,"num_write_kb":434880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":5037}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.115999+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:28:01.115999+0000","last_peered":"2024-08-17T17:28:01.115999+0000","last_clean":"2024-08-17T17:28:01.115999+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:28:01.115999+0000","last_undegraded":"2024-08-17T17:28:01.115999+0000","last_fullsized":"2024-08-17T17:28:01.115999+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":96,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.116903+0000","last_change":"2024-08-17T17:27:36.176161+0000","last_active":"2024-08-17T17:28:01.116903+0000","last_peered":"2024-08-17T17:28:01.116903+0000","last_clean":"2024-08-17T17:27:34.153378+0000","last_became_active":"2024-08-17T17:27:36.176161+0000","last_became_peered":"2024-08-17T17:27:36.176161+0000","last_unstale":"2024-08-17T17:28:01.116903+0000","last_undegraded":"2024-08-17T17:28:01.116903+0000","last_fullsized":"2024-08-17T17:27:36.151362+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:06.264054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"205'8478","reported_seq":8575,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:14.261895+0000","last_change":"2024-08-17T17:26:02.536813+0000","last_active":"2024-08-17T17:28:14.261895+0000","last_peered":"2024-08-17T17:28:14.261895+0000","last_clean":"2024-08-17T17:28:14.261895+0000","last_became_active":"2024-08-17T17:26:02.535602+0000","last_became_peered":"2024-08-17T17:26:02.535602+0000","last_unstale":"2024-08-17T17:28:14.261895+0000","last_undegraded":"2024-08-17T17:28:14.261895+0000","last_fullsized":"2024-08-17T17:28:14.261895+0000","mapping_epoch":167,"log_start":"174'1800","ondisk_log_start":"174'1800","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:17:50.650662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328138776,"num_objects":5008,"num_object_clones":0,"num_object_copies":15024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5008,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8478,"num_write_kb":431489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":267,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.164196+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:28:10.164196+0000","last_peered":"2024-08-17T17:28:10.164196+0000","last_clean":"2024-08-17T17:28:10.164196+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:28:10.164196+0000","last_undegraded":"2024-08-17T17:28:10.164196+0000","last_fullsized":"2024-08-17T17:28:10.164196+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":254,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.165506+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:28:10.165506+0000","last_peered":"2024-08-17T17:28:10.165506+0000","last_clean":"2024-08-17T17:28:10.165506+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:28:10.165506+0000","last_undegraded":"2024-08-17T17:28:10.165506+0000","last_fullsized":"2024-08-17T17:28:10.165506+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"205'5994","reported_seq":6158,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.875469+0000","last_change":"2024-08-17T17:27:33.999277+0000","last_active":"2024-08-17T17:28:13.875469+0000","last_peered":"2024-08-17T17:28:13.875469+0000","last_clean":"2024-08-17T17:28:13.875469+0000","last_became_active":"2024-08-17T17:26:42.109279+0000","last_became_peered":"2024-08-17T17:26:42.109279+0000","last_unstale":"2024-08-17T17:28:13.875469+0000","last_undegraded":"2024-08-17T17:28:13.875469+0000","last_fullsized":"2024-08-17T17:28:13.875469+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"198'4871","last_scrub_stamp":"2024-08-17T17:27:33.998966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:33.998966+0000","objects_scrubbed":3211,"log_size":5994,"log_dups_size":0,"ondisk_log_size":5994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:57.409751+0000","scrub_duration":703,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166330368,"num_objects":2538,"num_object_clones":0,"num_object_copies":7614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1125,"num_write_kb":14464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":507,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:01.106686+0000","last_change":"2024-08-17T17:27:36.158454+0000","last_active":"2024-08-17T17:28:01.106686+0000","last_peered":"2024-08-17T17:28:01.106686+0000","last_clean":"2024-08-17T17:27:34.151184+0000","last_became_active":"2024-08-17T17:27:36.158454+0000","last_became_peered":"2024-08-17T17:27:36.158454+0000","last_unstale":"2024-08-17T17:28:01.106686+0000","last_undegraded":"2024-08-17T17:27:36.152383+0000","last_fullsized":"2024-08-17T17:27:36.152283+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:54.606297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":782,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.167958+0000","last_change":"2024-08-17T17:22:38.211750+0000","last_active":"2024-08-17T17:28:10.167958+0000","last_peered":"2024-08-17T17:28:10.167958+0000","last_clean":"2024-08-17T17:28:10.167958+0000","last_became_active":"2024-08-17T17:22:38.210605+0000","last_became_peered":"2024-08-17T17:22:38.210605+0000","last_unstale":"2024-08-17T17:28:10.167958+0000","last_undegraded":"2024-08-17T17:28:10.167958+0000","last_fullsized":"2024-08-17T17:28:10.167958+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:54.086315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":228,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.159929+0000","last_change":"2024-08-17T17:27:36.164354+0000","last_active":"2024-08-17T17:28:10.159929+0000","last_peered":"2024-08-17T17:28:10.159929+0000","last_clean":"2024-08-17T17:27:34.153501+0000","last_became_active":"2024-08-17T17:27:36.164354+0000","last_became_peered":"2024-08-17T17:27:36.164354+0000","last_unstale":"2024-08-17T17:28:10.159929+0000","last_undegraded":"2024-08-17T17:28:10.159929+0000","last_fullsized":"2024-08-17T17:27:36.154854+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:37.962220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.116441+0000","last_change":"2024-08-17T17:27:36.153705+0000","last_active":"2024-08-17T17:28:01.116441+0000","last_peered":"2024-08-17T17:28:01.116441+0000","last_clean":"2024-08-17T17:22:13.179329+0000","last_became_active":"2024-08-17T17:27:36.153705+0000","last_became_peered":"2024-08-17T17:27:36.153705+0000","last_unstale":"2024-08-17T17:28:01.116441+0000","last_undegraded":"2024-08-17T17:28:01.116441+0000","last_fullsized":"2024-08-17T17:27:36.150850+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:19.689789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"205'6008","reported_seq":6175,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:14.310223+0000","last_change":"2024-08-17T17:27:36.200612+0000","last_active":"2024-08-17T17:28:14.310223+0000","last_peered":"2024-08-17T17:28:14.310223+0000","last_clean":"2024-08-17T17:27:34.856171+0000","last_became_active":"2024-08-17T17:27:36.200612+0000","last_became_peered":"2024-08-17T17:27:36.200612+0000","last_unstale":"2024-08-17T17:28:14.310223+0000","last_undegraded":"2024-08-17T17:27:36.158743+0000","last_fullsized":"2024-08-17T17:27:36.158630+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"198'4876","last_scrub_stamp":"2024-08-17T17:27:33.961415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:33.961415+0000","objects_scrubbed":3190,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:44.671505+0000","scrub_duration":878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164167680,"num_objects":2505,"num_object_clones":0,"num_object_copies":7515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2505,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1139,"num_write_kb":14528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":2505}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":578,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.106569+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:28:01.106569+0000","last_peered":"2024-08-17T17:28:01.106569+0000","last_clean":"2024-08-17T17:28:01.106569+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:28:01.106569+0000","last_undegraded":"2024-08-17T17:28:01.106569+0000","last_fullsized":"2024-08-17T17:28:01.106569+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":272,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.170619+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:28:10.170619+0000","last_peered":"2024-08-17T17:28:10.170619+0000","last_clean":"2024-08-17T17:28:10.170619+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:28:10.170619+0000","last_undegraded":"2024-08-17T17:28:10.170619+0000","last_fullsized":"2024-08-17T17:28:10.170619+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":269,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.174513+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:28:10.174513+0000","last_peered":"2024-08-17T17:28:10.174513+0000","last_clean":"2024-08-17T17:28:10.174513+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:28:10.174513+0000","last_undegraded":"2024-08-17T17:28:10.174513+0000","last_fullsized":"2024-08-17T17:28:10.174513+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"205'5902","reported_seq":5966,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:11.005407+0000","last_change":"2024-08-17T17:27:36.164263+0000","last_active":"2024-08-17T17:28:11.005407+0000","last_peered":"2024-08-17T17:28:11.005407+0000","last_clean":"2024-08-17T17:27:34.827973+0000","last_became_active":"2024-08-17T17:27:36.164263+0000","last_became_peered":"2024-08-17T17:27:36.164263+0000","last_unstale":"2024-08-17T17:28:11.005407+0000","last_undegraded":"2024-08-17T17:27:36.153053+0000","last_fullsized":"2024-08-17T17:27:36.152947+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169869312,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2592,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4218,"num_write_kb":217920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2592}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":668,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.168096+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:28:10.168096+0000","last_peered":"2024-08-17T17:28:10.168096+0000","last_clean":"2024-08-17T17:28:10.168096+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:28:10.168096+0000","last_undegraded":"2024-08-17T17:28:10.168096+0000","last_fullsized":"2024-08-17T17:28:10.168096+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.176404+0000","last_change":"2024-08-17T17:27:36.176565+0000","last_active":"2024-08-17T17:28:10.176404+0000","last_peered":"2024-08-17T17:28:10.176404+0000","last_clean":"2024-08-17T17:27:34.150964+0000","last_became_active":"2024-08-17T17:27:36.176565+0000","last_became_peered":"2024-08-17T17:27:36.176565+0000","last_unstale":"2024-08-17T17:28:10.176404+0000","last_undegraded":"2024-08-17T17:28:10.176404+0000","last_fullsized":"2024-08-17T17:27:36.158516+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:38:03.408152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":289,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.178207+0000","last_change":"2024-08-17T17:22:35.157060+0000","last_active":"2024-08-17T17:28:10.178207+0000","last_peered":"2024-08-17T17:28:10.178207+0000","last_clean":"2024-08-17T17:28:10.178207+0000","last_became_active":"2024-08-17T17:22:35.156650+0000","last_became_peered":"2024-08-17T17:22:35.156650+0000","last_unstale":"2024-08-17T17:28:10.178207+0000","last_undegraded":"2024-08-17T17:28:10.178207+0000","last_fullsized":"2024-08-17T17:28:10.178207+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:59.566999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"205'5970","reported_seq":6099,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.805286+0000","last_change":"2024-08-17T17:28:12.943254+0000","last_active":"2024-08-17T17:28:13.805286+0000","last_peered":"2024-08-17T17:28:13.805286+0000","last_clean":"2024-08-17T17:28:13.805286+0000","last_became_active":"2024-08-17T17:26:42.136378+0000","last_became_peered":"2024-08-17T17:26:42.136378+0000","last_unstale":"2024-08-17T17:28:13.805286+0000","last_undegraded":"2024-08-17T17:28:13.805286+0000","last_fullsized":"2024-08-17T17:28:13.805286+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:28:17.903507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166854656,"num_objects":2546,"num_object_clones":0,"num_object_copies":7638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4274,"num_write_kb":218240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1100,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:10.175944+0000","last_change":"2024-08-17T17:27:36.185858+0000","last_active":"2024-08-17T17:28:10.175944+0000","last_peered":"2024-08-17T17:28:10.175944+0000","last_clean":"2024-08-17T17:27:34.150914+0000","last_became_active":"2024-08-17T17:27:36.185858+0000","last_became_peered":"2024-08-17T17:27:36.185858+0000","last_unstale":"2024-08-17T17:28:10.175944+0000","last_undegraded":"2024-08-17T17:27:36.169544+0000","last_fullsized":"2024-08-17T17:27:36.167623+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:23:06.756208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.167614+0000","last_change":"2024-08-17T17:27:36.165070+0000","last_active":"2024-08-17T17:28:10.167614+0000","last_peered":"2024-08-17T17:28:10.167614+0000","last_clean":"2024-08-17T17:27:34.159785+0000","last_became_active":"2024-08-17T17:27:36.165070+0000","last_became_peered":"2024-08-17T17:27:36.165070+0000","last_unstale":"2024-08-17T17:28:10.167614+0000","last_undegraded":"2024-08-17T17:28:10.167614+0000","last_fullsized":"2024-08-17T17:27:36.153509+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:36.585084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":154,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176423+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:28:10.176423+0000","last_peered":"2024-08-17T17:28:10.176423+0000","last_clean":"2024-08-17T17:28:10.176423+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:28:10.176423+0000","last_undegraded":"2024-08-17T17:28:10.176423+0000","last_fullsized":"2024-08-17T17:28:10.176423+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"205'5839","reported_seq":6011,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.521811+0000","last_change":"2024-08-17T17:27:36.972727+0000","last_active":"2024-08-17T17:28:13.521811+0000","last_peered":"2024-08-17T17:28:13.521811+0000","last_clean":"2024-08-17T17:28:13.521811+0000","last_became_active":"2024-08-17T17:26:42.111775+0000","last_became_peered":"2024-08-17T17:26:42.111775+0000","last_unstale":"2024-08-17T17:28:13.521811+0000","last_undegraded":"2024-08-17T17:28:13.521811+0000","last_fullsized":"2024-08-17T17:28:13.521811+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'4781","last_scrub_stamp":"2024-08-17T17:27:36.972567+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:36.972567+0000","objects_scrubbed":3082,"log_size":5839,"log_dups_size":0,"ondisk_log_size":5839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:09.455827+0000","scrub_duration":720,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383552,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1058,"num_write_kb":13056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":235,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.160718+0000","last_change":"2024-08-17T17:27:36.161497+0000","last_active":"2024-08-17T17:28:10.160718+0000","last_peered":"2024-08-17T17:28:10.160718+0000","last_clean":"2024-08-17T17:27:34.154337+0000","last_became_active":"2024-08-17T17:27:36.161497+0000","last_became_peered":"2024-08-17T17:27:36.161497+0000","last_unstale":"2024-08-17T17:28:10.160718+0000","last_undegraded":"2024-08-17T17:28:10.160718+0000","last_fullsized":"2024-08-17T17:27:36.153746+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:12.395407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":553,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176715+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:28:10.176715+0000","last_peered":"2024-08-17T17:28:10.176715+0000","last_clean":"2024-08-17T17:28:10.176715+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:28:10.176715+0000","last_undegraded":"2024-08-17T17:28:10.176715+0000","last_fullsized":"2024-08-17T17:28:10.176715+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176446+0000","last_change":"2024-08-17T17:22:35.156003+0000","last_active":"2024-08-17T17:28:10.176446+0000","last_peered":"2024-08-17T17:28:10.176446+0000","last_clean":"2024-08-17T17:28:10.176446+0000","last_became_active":"2024-08-17T17:22:35.155653+0000","last_became_peered":"2024-08-17T17:22:35.155653+0000","last_unstale":"2024-08-17T17:28:10.176446+0000","last_undegraded":"2024-08-17T17:28:10.176446+0000","last_fullsized":"2024-08-17T17:28:10.176446+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:44.965126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"205'5807","reported_seq":5933,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:14.242587+0000","last_change":"2024-08-17T17:27:36.176478+0000","last_active":"2024-08-17T17:28:14.242587+0000","last_peered":"2024-08-17T17:28:14.242587+0000","last_clean":"2024-08-17T17:27:34.777586+0000","last_became_active":"2024-08-17T17:27:36.176478+0000","last_became_peered":"2024-08-17T17:27:36.176478+0000","last_unstale":"2024-08-17T17:28:14.242587+0000","last_undegraded":"2024-08-17T17:27:36.158634+0000","last_fullsized":"2024-08-17T17:27:36.158532+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5807,"log_dups_size":0,"ondisk_log_size":5807,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:28:14.967192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160399360,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2447,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4179,"num_write_kb":212064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2447}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":256,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176624+0000","last_change":"2024-08-17T17:22:35.159454+0000","last_active":"2024-08-17T17:28:10.176624+0000","last_peered":"2024-08-17T17:28:10.176624+0000","last_clean":"2024-08-17T17:28:10.176624+0000","last_became_active":"2024-08-17T17:22:35.157827+0000","last_became_peered":"2024-08-17T17:22:35.157827+0000","last_unstale":"2024-08-17T17:28:10.176624+0000","last_undegraded":"2024-08-17T17:28:10.176624+0000","last_fullsized":"2024-08-17T17:28:10.176624+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:22.914140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":768,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.107280+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:28:01.107280+0000","last_peered":"2024-08-17T17:28:01.107280+0000","last_clean":"2024-08-17T17:28:01.107280+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:28:01.107280+0000","last_undegraded":"2024-08-17T17:28:01.107280+0000","last_fullsized":"2024-08-17T17:28:01.107280+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":254,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176351+0000","last_change":"2024-08-17T17:22:35.158040+0000","last_active":"2024-08-17T17:28:10.176351+0000","last_peered":"2024-08-17T17:28:10.176351+0000","last_clean":"2024-08-17T17:28:10.176351+0000","last_became_active":"2024-08-17T17:22:35.154782+0000","last_became_peered":"2024-08-17T17:22:35.154782+0000","last_unstale":"2024-08-17T17:28:10.176351+0000","last_undegraded":"2024-08-17T17:28:10.176351+0000","last_fullsized":"2024-08-17T17:28:10.176351+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:58.755222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"205'5747","reported_seq":5878,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:11.503515+0000","last_change":"2024-08-17T17:27:33.729902+0000","last_active":"2024-08-17T17:28:11.503515+0000","last_peered":"2024-08-17T17:28:11.503515+0000","last_clean":"2024-08-17T17:28:11.503515+0000","last_became_active":"2024-08-17T17:26:42.111242+0000","last_became_peered":"2024-08-17T17:26:42.111242+0000","last_unstale":"2024-08-17T17:28:11.503515+0000","last_undegraded":"2024-08-17T17:28:11.503515+0000","last_fullsized":"2024-08-17T17:28:11.503515+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"198'4761","last_scrub_stamp":"2024-08-17T17:27:33.729725+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:33.729725+0000","objects_scrubbed":3122,"log_size":5747,"log_dups_size":0,"ondisk_log_size":5747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:30.060610+0000","scrub_duration":0.51469149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167510016,"num_objects":2556,"num_object_clones":0,"num_object_copies":7668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":992,"num_write_kb":13632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":265,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.165711+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:28:10.165711+0000","last_peered":"2024-08-17T17:28:10.165711+0000","last_clean":"2024-08-17T17:28:10.165711+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:28:10.165711+0000","last_undegraded":"2024-08-17T17:28:10.165711+0000","last_fullsized":"2024-08-17T17:28:10.165711+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":776,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.175614+0000","last_change":"2024-08-17T17:27:36.171574+0000","last_active":"2024-08-17T17:28:10.175614+0000","last_peered":"2024-08-17T17:28:10.175614+0000","last_clean":"2024-08-17T17:27:34.151904+0000","last_became_active":"2024-08-17T17:27:36.171574+0000","last_became_peered":"2024-08-17T17:27:36.171574+0000","last_unstale":"2024-08-17T17:28:10.175614+0000","last_undegraded":"2024-08-17T17:28:10.175614+0000","last_fullsized":"2024-08-17T17:27:36.158328+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:56.371461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.162582+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:28:10.162582+0000","last_peered":"2024-08-17T17:28:10.162582+0000","last_clean":"2024-08-17T17:28:10.162582+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:28:10.162582+0000","last_undegraded":"2024-08-17T17:28:10.162582+0000","last_fullsized":"2024-08-17T17:28:10.162582+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"205'5745","reported_seq":5808,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:10.968588+0000","last_change":"2024-08-17T17:27:36.157616+0000","last_active":"2024-08-17T17:28:10.968588+0000","last_peered":"2024-08-17T17:28:10.968588+0000","last_clean":"2024-08-17T17:27:34.918889+0000","last_became_active":"2024-08-17T17:27:36.157616+0000","last_became_peered":"2024-08-17T17:27:36.157616+0000","last_unstale":"2024-08-17T17:28:10.968588+0000","last_undegraded":"2024-08-17T17:27:36.152704+0000","last_fullsized":"2024-08-17T17:27:36.152619+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174522368,"num_objects":2663,"num_object_clones":0,"num_object_copies":7989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4103,"num_write_kb":216512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2663}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":284,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.174261+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:28:10.174261+0000","last_peered":"2024-08-17T17:28:10.174261+0000","last_clean":"2024-08-17T17:28:10.174261+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:28:10.174261+0000","last_undegraded":"2024-08-17T17:28:10.174261+0000","last_fullsized":"2024-08-17T17:28:10.174261+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":315,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.107196+0000","last_change":"2024-08-17T17:27:36.160014+0000","last_active":"2024-08-17T17:28:01.107196+0000","last_peered":"2024-08-17T17:28:01.107196+0000","last_clean":"2024-08-17T17:07:02.886086+0000","last_became_active":"2024-08-17T17:27:36.160014+0000","last_became_peered":"2024-08-17T17:27:36.160014+0000","last_unstale":"2024-08-17T17:28:01.107196+0000","last_undegraded":"2024-08-17T17:28:01.107196+0000","last_fullsized":"2024-08-17T17:27:36.153838+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:12.687947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.116153+0000","last_change":"2024-08-17T17:27:36.155245+0000","last_active":"2024-08-17T17:28:01.116153+0000","last_peered":"2024-08-17T17:28:01.116153+0000","last_clean":"2024-08-17T17:27:34.152502+0000","last_became_active":"2024-08-17T17:27:36.155245+0000","last_became_peered":"2024-08-17T17:27:36.155245+0000","last_unstale":"2024-08-17T17:28:01.116153+0000","last_undegraded":"2024-08-17T17:28:01.116153+0000","last_fullsized":"2024-08-17T17:27:36.150204+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:19:43.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.11","version":"205'5834","reported_seq":12477,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.880960+0000","last_change":"2024-08-17T17:27:35.611136+0000","last_active":"2024-08-17T17:28:13.880960+0000","last_peered":"2024-08-17T17:28:13.880960+0000","last_clean":"2024-08-17T17:28:13.880960+0000","last_became_active":"2024-08-17T17:27:35.163188+0000","last_became_peered":"2024-08-17T17:27:35.163188+0000","last_unstale":"2024-08-17T17:28:13.880960+0000","last_undegraded":"2024-08-17T17:28:13.880960+0000","last_fullsized":"2024-08-17T17:28:13.880960+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"200'4792","last_scrub_stamp":"2024-08-17T17:27:35.610908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:35.610908+0000","objects_scrubbed":3165,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:48.612736+0000","scrub_duration":330,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1042,"num_write_kb":11072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":96,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.116530+0000","last_change":"2024-08-17T17:27:36.166803+0000","last_active":"2024-08-17T17:28:01.116530+0000","last_peered":"2024-08-17T17:28:01.116530+0000","last_clean":"2024-08-17T17:27:34.153014+0000","last_became_active":"2024-08-17T17:27:36.166803+0000","last_became_peered":"2024-08-17T17:27:36.166803+0000","last_unstale":"2024-08-17T17:28:01.116530+0000","last_undegraded":"2024-08-17T17:28:01.116530+0000","last_fullsized":"2024-08-17T17:27:36.150441+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:36:39.160270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":106,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:01.118821+0000","last_change":"2024-08-17T17:27:36.153883+0000","last_active":"2024-08-17T17:28:01.118821+0000","last_peered":"2024-08-17T17:28:01.118821+0000","last_clean":"2024-08-17T17:27:34.152919+0000","last_became_active":"2024-08-17T17:27:36.153883+0000","last_became_peered":"2024-08-17T17:27:36.153883+0000","last_unstale":"2024-08-17T17:28:01.118821+0000","last_undegraded":"2024-08-17T17:27:36.150733+0000","last_fullsized":"2024-08-17T17:27:36.150628+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:17.872426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.10","version":"205'5882","reported_seq":11792,"reported_epoch":205,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:28:13.612167+0000","last_change":"2024-08-17T17:27:37.213003+0000","last_active":"2024-08-17T17:28:13.612167+0000","last_peered":"2024-08-17T17:28:13.612167+0000","last_clean":"2024-08-17T17:27:34.911262+0000","last_became_active":"2024-08-17T17:27:37.207847+0000","last_became_peered":"2024-08-17T17:27:37.207847+0000","last_unstale":"2024-08-17T17:28:13.612167+0000","last_undegraded":"2024-08-17T17:27:37.173791+0000","last_fullsized":"2024-08-17T17:28:13.612167+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5882,"log_dups_size":0,"ondisk_log_size":5882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1258,"num_objects_misplaced":1198,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4218,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1929,"num_bytes_recovered":126418944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,2],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1","objects":1258},{"shards":"0,1,2","objects":1198}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":252,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.176153+0000","last_change":"2024-08-17T17:27:36.176571+0000","last_active":"2024-08-17T17:28:10.176153+0000","last_peered":"2024-08-17T17:28:10.176153+0000","last_clean":"2024-08-17T17:27:34.150999+0000","last_became_active":"2024-08-17T17:27:36.176571+0000","last_became_peered":"2024-08-17T17:27:36.176571+0000","last_unstale":"2024-08-17T17:28:10.176153+0000","last_undegraded":"2024-08-17T17:28:10.176153+0000","last_fullsized":"2024-08-17T17:27:36.172432+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:35.412782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.107023+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:28:01.107023+0000","last_peered":"2024-08-17T17:28:01.107023+0000","last_clean":"2024-08-17T17:28:01.107023+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:28:01.107023+0000","last_undegraded":"2024-08-17T17:28:01.107023+0000","last_fullsized":"2024-08-17T17:28:01.107023+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.13","version":"205'5806","reported_seq":9493,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:28:13.241199+0000","last_change":"2024-08-17T17:27:37.247788+0000","last_active":"2024-08-17T17:28:13.241199+0000","last_peered":"2024-08-17T17:28:13.241199+0000","last_clean":"2024-08-17T17:26:58.141600+0000","last_became_active":"2024-08-17T17:27:37.199531+0000","last_became_peered":"2024-08-17T17:27:37.199531+0000","last_unstale":"2024-08-17T17:28:13.241199+0000","last_undegraded":"2024-08-17T17:27:37.195452+0000","last_fullsized":"2024-08-17T17:27:37.153269+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161873920,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2470,"num_objects_misplaced":1994,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4164,"num_write_kb":212288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1200,"num_bytes_recovered":78643200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[2,0],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2","objects":1473},{"shards":"2,7","objects":997}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":135,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.106891+0000","last_change":"2024-08-17T17:22:35.149514+0000","last_active":"2024-08-17T17:28:01.106891+0000","last_peered":"2024-08-17T17:28:01.106891+0000","last_clean":"2024-08-17T17:28:01.106891+0000","last_became_active":"2024-08-17T17:22:35.149176+0000","last_became_peered":"2024-08-17T17:22:35.149176+0000","last_unstale":"2024-08-17T17:28:01.106891+0000","last_undegraded":"2024-08-17T17:28:01.106891+0000","last_fullsized":"2024-08-17T17:28:01.106891+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:10.518331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":221,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.161032+0000","last_change":"2024-08-17T17:27:36.163742+0000","last_active":"2024-08-17T17:28:10.161032+0000","last_peered":"2024-08-17T17:28:10.161032+0000","last_clean":"2024-08-17T17:27:34.154361+0000","last_became_active":"2024-08-17T17:27:36.163742+0000","last_became_peered":"2024-08-17T17:27:36.163742+0000","last_unstale":"2024-08-17T17:28:10.161032+0000","last_undegraded":"2024-08-17T17:28:10.161032+0000","last_fullsized":"2024-08-17T17:27:36.153948+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:33:26.610976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.12","version":"205'5791","reported_seq":10222,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.939810+0000","last_change":"2024-08-17T17:27:21.309750+0000","last_active":"2024-08-17T17:28:10.939810+0000","last_peered":"2024-08-17T17:28:10.939810+0000","last_clean":"2024-08-17T17:28:10.939810+0000","last_became_active":"2024-08-17T17:27:21.307613+0000","last_became_peered":"2024-08-17T17:27:21.307613+0000","last_unstale":"2024-08-17T17:28:10.939810+0000","last_undegraded":"2024-08-17T17:28:10.939810+0000","last_fullsized":"2024-08-17T17:28:10.939810+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281792,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4136,"num_write_kb":213056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2175,"num_bytes_recovered":142540800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.174853+0000","last_change":"2024-08-17T17:27:36.168085+0000","last_active":"2024-08-17T17:28:10.174853+0000","last_peered":"2024-08-17T17:28:10.174853+0000","last_clean":"2024-08-17T17:27:34.156630+0000","last_became_active":"2024-08-17T17:27:36.168085+0000","last_became_peered":"2024-08-17T17:27:36.168085+0000","last_unstale":"2024-08-17T17:28:10.174853+0000","last_undegraded":"2024-08-17T17:28:10.174853+0000","last_fullsized":"2024-08-17T17:27:36.153457+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:06:39.295085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":270,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.162344+0000","last_change":"2024-08-17T17:27:36.164472+0000","last_active":"2024-08-17T17:28:10.162344+0000","last_peered":"2024-08-17T17:28:10.162344+0000","last_clean":"2024-08-17T17:27:34.153209+0000","last_became_active":"2024-08-17T17:27:36.164472+0000","last_became_peered":"2024-08-17T17:27:36.164472+0000","last_unstale":"2024-08-17T17:28:10.162344+0000","last_undegraded":"2024-08-17T17:28:10.162344+0000","last_fullsized":"2024-08-17T17:27:36.155839+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:11.077063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.178343+0000","last_change":"2024-08-17T17:27:36.171242+0000","last_active":"2024-08-17T17:28:10.178343+0000","last_peered":"2024-08-17T17:28:10.178343+0000","last_clean":"2024-08-17T17:22:32.117919+0000","last_became_active":"2024-08-17T17:27:36.171242+0000","last_became_peered":"2024-08-17T17:27:36.171242+0000","last_unstale":"2024-08-17T17:28:10.178343+0000","last_undegraded":"2024-08-17T17:28:10.178343+0000","last_fullsized":"2024-08-17T17:27:36.158077+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:27.403782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":281,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.162712+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:28:10.162712+0000","last_peered":"2024-08-17T17:28:10.162712+0000","last_clean":"2024-08-17T17:28:10.162712+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:28:10.162712+0000","last_undegraded":"2024-08-17T17:28:10.162712+0000","last_fullsized":"2024-08-17T17:28:10.162712+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":258,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.174442+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:28:10.174442+0000","last_peered":"2024-08-17T17:28:10.174442+0000","last_clean":"2024-08-17T17:28:10.174442+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:28:10.174442+0000","last_undegraded":"2024-08-17T17:28:10.174442+0000","last_fullsized":"2024-08-17T17:28:10.174442+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":262,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.174677+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:28:10.174677+0000","last_peered":"2024-08-17T17:28:10.174677+0000","last_clean":"2024-08-17T17:28:10.174677+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:28:10.174677+0000","last_undegraded":"2024-08-17T17:28:10.174677+0000","last_fullsized":"2024-08-17T17:28:10.174677+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":278,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.160405+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:28:10.160405+0000","last_peered":"2024-08-17T17:28:10.160405+0000","last_clean":"2024-08-17T17:28:10.160405+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:28:10.160405+0000","last_undegraded":"2024-08-17T17:28:10.160405+0000","last_fullsized":"2024-08-17T17:28:10.160405+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.116315+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:28:01.116315+0000","last_peered":"2024-08-17T17:28:01.116315+0000","last_clean":"2024-08-17T17:28:01.116315+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:28:01.116315+0000","last_undegraded":"2024-08-17T17:28:01.116315+0000","last_fullsized":"2024-08-17T17:28:01.116315+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":286,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.168258+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:28:10.168258+0000","last_peered":"2024-08-17T17:28:10.168258+0000","last_clean":"2024-08-17T17:28:10.168258+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:28:10.168258+0000","last_undegraded":"2024-08-17T17:28:10.168258+0000","last_fullsized":"2024-08-17T17:28:10.168258+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":264,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.176249+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:28:10.176249+0000","last_peered":"2024-08-17T17:28:10.176249+0000","last_clean":"2024-08-17T17:28:10.176249+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:28:10.176249+0000","last_undegraded":"2024-08-17T17:28:10.176249+0000","last_fullsized":"2024-08-17T17:28:10.176249+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"205'5876","reported_seq":6038,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:13.586071+0000","last_change":"2024-08-17T17:27:33.906245+0000","last_active":"2024-08-17T17:28:13.586071+0000","last_peered":"2024-08-17T17:28:13.586071+0000","last_clean":"2024-08-17T17:28:13.586071+0000","last_became_active":"2024-08-17T17:26:42.126197+0000","last_became_peered":"2024-08-17T17:26:42.126197+0000","last_unstale":"2024-08-17T17:28:13.586071+0000","last_undegraded":"2024-08-17T17:28:13.586071+0000","last_fullsized":"2024-08-17T17:28:13.586071+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"198'4766","last_scrub_stamp":"2024-08-17T17:27:33.906070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:27:33.906070+0000","objects_scrubbed":3131,"log_size":5876,"log_dups_size":0,"ondisk_log_size":5876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:06.451394+0000","scrub_duration":724,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162725888,"num_objects":2483,"num_object_clones":0,"num_object_copies":7449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1112,"num_write_kb":14848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.116392+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:28:01.116392+0000","last_peered":"2024-08-17T17:28:01.116392+0000","last_clean":"2024-08-17T17:28:01.116392+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:28:01.116392+0000","last_undegraded":"2024-08-17T17:28:01.116392+0000","last_fullsized":"2024-08-17T17:28:01.116392+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":197,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:01.118805+0000","last_change":"2024-08-17T17:27:36.163668+0000","last_active":"2024-08-17T17:28:01.118805+0000","last_peered":"2024-08-17T17:28:01.118805+0000","last_clean":"2024-08-17T17:27:34.152835+0000","last_became_active":"2024-08-17T17:27:36.163668+0000","last_became_peered":"2024-08-17T17:27:36.163668+0000","last_unstale":"2024-08-17T17:28:01.118805+0000","last_undegraded":"2024-08-17T17:27:36.150505+0000","last_fullsized":"2024-08-17T17:27:36.150405+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:21.064713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"205'5826","reported_seq":5883,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.821856+0000","last_change":"2024-08-17T17:26:42.161996+0000","last_active":"2024-08-17T17:28:10.821856+0000","last_peered":"2024-08-17T17:28:10.821856+0000","last_clean":"2024-08-17T17:28:10.821856+0000","last_became_active":"2024-08-17T17:26:42.160414+0000","last_became_peered":"2024-08-17T17:26:42.160414+0000","last_unstale":"2024-08-17T17:28:10.821856+0000","last_undegraded":"2024-08-17T17:28:10.821856+0000","last_fullsized":"2024-08-17T17:28:10.821856+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":176,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:01.510997+0000","last_clean_scrub_stamp":"2024-08-17T17:26:01.510997+0000","objects_scrubbed":0,"log_size":5826,"log_dups_size":0,"ondisk_log_size":5826,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170557440,"num_objects":2603,"num_object_clones":0,"num_object_copies":7809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4151,"num_write_kb":216096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-08-17T17:28:01.116206+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:28:01.116206+0000","last_peered":"2024-08-17T17:28:01.116206+0000","last_clean":"2024-08-17T17:28:01.116206+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:28:01.116206+0000","last_undegraded":"2024-08-17T17:28:01.116206+0000","last_fullsized":"2024-08-17T17:28:01.116206+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":182,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:01.116393+0000","last_change":"2024-08-17T17:27:36.166858+0000","last_active":"2024-08-17T17:28:01.116393+0000","last_peered":"2024-08-17T17:28:01.116393+0000","last_clean":"2024-08-17T17:27:34.152739+0000","last_became_active":"2024-08-17T17:27:36.166858+0000","last_became_peered":"2024-08-17T17:27:36.166858+0000","last_unstale":"2024-08-17T17:28:01.116393+0000","last_undegraded":"2024-08-17T17:27:36.150964+0000","last_fullsized":"2024-08-17T17:27:36.150848+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:25.954846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":283,"reported_epoch":205,"state":"active+clean","last_fresh":"2024-08-17T17:28:10.160021+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:28:10.160021+0000","last_peered":"2024-08-17T17:28:10.160021+0000","last_clean":"2024-08-17T17:28:10.160021+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:28:10.160021+0000","last_undegraded":"2024-08-17T17:28:10.160021+0000","last_fullsized":"2024-08-17T17:28:10.160021+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":276,"reported_epoch":205,"state":"active+undersized","last_fresh":"2024-08-17T17:28:10.161901+0000","last_change":"2024-08-17T17:27:36.160202+0000","last_active":"2024-08-17T17:28:10.161901+0000","last_peered":"2024-08-17T17:28:10.161901+0000","last_clean":"2024-08-17T17:27:34.154748+0000","last_became_active":"2024-08-17T17:27:36.160202+0000","last_became_peered":"2024-08-17T17:27:36.160202+0000","last_unstale":"2024-08-17T17:28:10.161901+0000","last_undegraded":"2024-08-17T17:28:10.161901+0000","last_fullsized":"2024-08-17T17:27:36.155435+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:35.895991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":204,"state":"active+undersized","last_fresh":"2024-08-17T17:28:01.115923+0000","last_change":"2024-08-17T17:27:36.156351+0000","last_active":"2024-08-17T17:28:01.115923+0000","last_peered":"2024-08-17T17:28:01.115923+0000","last_clean":"2024-08-17T17:27:34.152341+0000","last_became_active":"2024-08-17T17:27:36.156351+0000","last_became_peered":"2024-08-17T17:27:36.156351+0000","last_unstale":"2024-08-17T17:28:01.115923+0000","last_undegraded":"2024-08-17T17:28:01.115923+0000","last_fullsized":"2024-08-17T17:27:36.149941+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:07.514027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":279,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:28:10.174648+0000","last_change":"2024-08-17T17:27:36.160637+0000","last_active":"2024-08-17T17:28:10.174648+0000","last_peered":"2024-08-17T17:28:10.174648+0000","last_clean":"2024-08-17T17:27:34.156558+0000","last_became_active":"2024-08-17T17:27:36.160637+0000","last_became_peered":"2024-08-17T17:27:36.160637+0000","last_unstale":"2024-08-17T17:28:10.174648+0000","last_undegraded":"2024-08-17T17:27:36.153258+0000","last_fullsized":"2024-08-17T17:27:36.153154+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:51.606546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5278072856,"num_objects":80538,"num_object_clones":0,"num_object_copies":241614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27282,"num_objects_misplaced":4832,"num_objects_unfound":0,"num_objects_dirty":80538,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116161,"num_write_kb":5689473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9023,"num_bytes_recovered":591331328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17386123264,"data_stored":17386111048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157884,"ondisk_log_size":157884,"up":69,"acting":69,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":83,"acting":83,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":5700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":81,"acting":81,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":21,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":2,"acting":2,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201576,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2886084,"kb_used_data":2588344,"kb_used_omap":1,"kb_used_meta":297726,"kb_avail":90859068,"statfs":{"total":95995035648,"available":93039685632,"internally_reserved":0,"allocated":2650464256,"data_stored":2649352550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":304871883},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463206,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2593100,"kb_used_data":2417604,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":91152052,"statfs":{"total":95995035648,"available":93339701248,"internally_reserved":0,"allocated":2475626496,"data_stored":2474557478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179698119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":167503724850,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986486,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258300,"kb_used_data":1916272,"kb_used_omap":1,"kb_used_meta":342014,"kb_avail":91486852,"statfs":{"total":95995035648,"available":93682536448,"internally_reserved":0,"allocated":1962262528,"data_stored":1961158408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":350222790},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435378,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3573928,"kb_used_data":3249248,"kb_used_omap":5,"kb_used_meta":324602,"kb_avail":90171224,"statfs":{"total":95995035648,"available":92335333376,"internally_reserved":0,"allocated":3327229952,"data_stored":3326137486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":332392521},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598904,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3627052,"kb_used_data":3234404,"kb_used_omap":7,"kb_used_meta":392632,"kb_avail":90118100,"statfs":{"total":95995035648,"available":92280934400,"internally_reserved":0,"allocated":3312029696,"data_stored":3310933614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":402055341},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762431,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2328868,"kb_used_data":2116700,"kb_used_omap":7,"kb_used_meta":212152,"kb_avail":91416284,"statfs":{"total":95995035648,"available":93610274816,"internally_reserved":0,"allocated":2167500800,"data_stored":2166412040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":217244065},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925957,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883000,"kb_used_data":1536176,"kb_used_omap":7,"kb_used_meta":346808,"kb_avail":91862152,"statfs":{"total":95995035648,"available":94066843648,"internally_reserved":0,"allocated":1573044224,"data_stored":1571936678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":355131996},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1562447872,"data_stored":1562443800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2155024384,"data_stored":2155020312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3305246720,"data_stored":3305242648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3290693632,"data_stored":3290693632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1955004416,"data_stored":1955004416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2493251584,"data_stored":2493251584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2624454656,"data_stored":2624454656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:28:20.304 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:28:20.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_1 pg_num 36 2024-08-17T17:28:20.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:20.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:20.467 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:20 smithi064 ceph-mon[104516]: pgmap v625: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 26416/234123 objects degraded (11.283%); 4348/234123 objects misplaced (1.857%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:28:20.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:20 smithi064 ceph-mon[109246]: pgmap v625: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 26416/234123 objects degraded (11.283%); 4348/234123 objects misplaced (1.857%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:28:20.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:20.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:20.614 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:20 smithi175 ceph-mon[101434]: pgmap v625: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 26416/234123 objects degraded (11.283%); 4348/234123 objects misplaced (1.857%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:28:20.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:20.762 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:20.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:20.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:20.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:21.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:21.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:21.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:21.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:21.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:21.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:21.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:21.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:21.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:21.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:21.978 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:22.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:28:22.082 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:28:22.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:22.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:22.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:22.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:22.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:22.352 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:22.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:22 smithi064 ceph-mon[104516]: pgmap v626: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 25645/228255 objects degraded (11.235%); 3900/228255 objects misplaced (1.709%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:28:22.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:22 smithi064 ceph-mon[109246]: pgmap v626: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 25645/228255 objects degraded (11.235%); 3900/228255 objects misplaced (1.709%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:28:22.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:22.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:22.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:22.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:22.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:22 smithi175 ceph-mon[101434]: pgmap v626: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 25645/228255 objects degraded (11.235%); 3900/228255 objects misplaced (1.709%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:28:22.750 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:22.774 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.920 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:22.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:23.021 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:23.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:23.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:23.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:23.192 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144221 6.17 6:e927cd36:::benchmark_data_smithi064_185391_object36488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.174307+0000", 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010181851, 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010211813, 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144221 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174307+0000", 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174307+0000", 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174308+0000", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4169999999999999e-06 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174313+0000", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.972e-06 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174329+0000", 2024-08-17T17:28:23.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5345e-05 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174337+0000", 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0460000000000006e-06 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174387+0000", 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0031999999999998e-05 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:28:23.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174508+0000", 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012099699999999999 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.174935+0000", 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042702899999999999 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35007.0:144221 6.17 e205/176)", 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.179916+0000", 2024-08-17T17:28:23.197 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0045727600000000004, 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004679267, 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179916+0000", 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179916+0000", 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179917+0000", 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4900000000000004e-07 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179922+0000", 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179936+0000", 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.382e-05 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.179943+0000", 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3810000000000001e-06 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144225 6.11 6:89139d4b:::benchmark_data_smithi064_185391_object36492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.200 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.180438+0000", 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0040508999999999996, 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041999899999999998, 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.201 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144225 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180438+0000", 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180438+0000", 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180439+0000", 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5600000000000001e-06 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180445+0000", 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.648e-06 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180468+0000", 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2795000000000001e-05 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180477+0000", 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1009999999999998e-06 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180585+0000", 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108146 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180729+0000", 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014394500000000001 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181995+0000", 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001265565 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.205 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144227 6.11 6:8ead7033:::benchmark_data_smithi064_185391_object36494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.180549+0000", 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0039399489999999999, 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004148466, 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144227 2024-08-17T17:28:23.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180549+0000", 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180549+0000", 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180549+0000", 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2200000000000004e-07 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180553+0000", 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2150000000000001e-06 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180562+0000", 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6849999999999996e-06 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180569+0000", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.409e-06 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183869+0000", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032998760000000002 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183991+0000", 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121884 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.184219+0000", 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022834099999999999 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144228 6.17 6.647c857 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.180578+0000", 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0039105429999999998, 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041765810000000004, 2024-08-17T17:28:23.210 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144228 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180578+0000", 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180578+0000", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180579+0000", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9900000000000001e-07 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180581+0000", 2024-08-17T17:28:23.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.695e-06 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180588+0000", 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0160000000000003e-06 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180594+0000", 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3979999999999998e-06 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144231 6.b 6.65e0bf7b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.180602+0000", 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0038869680000000002, 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041955619999999999, 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144231 2024-08-17T17:28:23.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180602+0000", 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180602+0000", 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180602+0000", 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2600000000000002e-07 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180605+0000", 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2819999999999999e-06 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180611+0000", 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8410000000000002e-06 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180617+0000", 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3179999999999997e-06 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:144232 6.16 6:699191de:::benchmark_data_smithi064_185391_object36499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.180632+0000", 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0038569799999999999, 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042101869999999998, 2024-08-17T17:28:23.217 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 144232 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180632+0000", 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180632+0000", 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180632+0000", 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7e-07 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180635+0000", 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6649999999999999e-06 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180642+0000", 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2350000000000002e-06 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180647+0000", 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1950000000000002e-06 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180743+0000", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5544000000000005e-05 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.180859+0000", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011638599999999999 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181253+0000", 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039343199999999998 2024-08-17T17:28:23.221 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35007.0:144221 6.17 e205/176)", 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.181104+0000", 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0033845070000000001, 2024-08-17T17:28:23.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037990960000000001, 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181104+0000", 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181104+0000", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181105+0000", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8700000000000005e-07 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181109+0000", 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0890000000000002e-06 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181120+0000", 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0528e-05 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181126+0000", 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4690000000000002e-06 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(6.13 205/201 [PushOp(6:ce4c955b:::benchmark_data_smithi064_185391_object49734:head, version: 174'3067, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:ce4c955b:::benchmark_data_smithi064_185391_object49734:head@174'3067, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:28:23.225 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.181150+0000", 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.003338644, 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038246069999999998, 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181150+0000", 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181150+0000", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181150+0000", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7899999999999999e-07 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181183+0000", 2024-08-17T17:28:23.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2131000000000001e-05 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181199+0000", 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6473999999999998e-05 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.181214+0000", 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4453999999999999e-05 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35007.0:144232 6.16 e205/176)", 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:23.183777+0000", 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00071218500000000005, 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012333019999999999, 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183777+0000", 2024-08-17T17:28:23.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183777+0000", 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183777+0000", 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4000000000000001e-07 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183782+0000", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4150000000000001e-06 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183791+0000", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.764e-06 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:23.183798+0000", 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8839999999999997e-06 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.231 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 10 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:28:23.232 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:23.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:23.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:23.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:23.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25645/228255 objects degraded (11.235%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:23.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25645/228255 objects degraded (11.235%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:23.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:23.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:23.601 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25645/228255 objects degraded (11.235%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:23.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:23.726 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:23.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:23.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:23.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:23.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:23.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:24.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:28:24.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:24.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:28:24.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:24.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:24.218 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:23] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:28:24.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:24.243 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:28:24.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:24.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:24.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:24.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:24.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:24 smithi064 ceph-mon[104516]: pgmap v627: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 25230/224430 objects degraded (11.242%); 3900/224430 objects misplaced (1.738%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:24.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:24.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:24 smithi064 ceph-mon[109246]: pgmap v627: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 25230/224430 objects degraded (11.242%); 3900/224430 objects misplaced (1.738%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:24.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:24.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:24 smithi175 ceph-mon[101434]: pgmap v627: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 25230/224430 objects degraded (11.242%); 3900/224430 objects misplaced (1.738%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:28:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:24.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:24.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:24.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:24.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:24.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:24.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:25.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:25.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:25.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:25.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:25.418 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:25.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:25.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:25.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: osdmap e206: 8 total, 7 up, 7 in 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: osdmap e206: 8 total, 7 up, 7 in 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:25.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:25.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/676245540' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: osdmap e206: 8 total, 7 up, 7 in 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:25.684 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:25.786 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:25.793 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:25.809 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:25.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:25.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:26.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:26.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:26.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:26.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:26.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:26.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:26 smithi064 ceph-mon[104516]: pgmap v629: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 23886/214359 objects degraded (11.143%); 2823/214359 objects misplaced (1.317%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:26.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:26 smithi064 ceph-mon[109246]: pgmap v629: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 23886/214359 objects degraded (11.143%); 2823/214359 objects misplaced (1.317%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:26.528 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:26 smithi175 ceph-mon[101434]: pgmap v629: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 23886/214359 objects degraded (11.143%); 2823/214359 objects misplaced (1.317%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:26.623 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:26.630 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:26.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:26.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:26.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:26.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:26.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:27.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:27.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:27.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:27.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:27.271 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pg_num to 36 2024-08-17T17:28:27.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:27.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1688150597' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:27 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:27.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:27.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:27.620 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1688150597' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:27 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1688150597' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:27 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:28:27.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:27.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:27.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:28:27.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:27.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:28.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.023 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:28.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:28.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:28.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:28.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:28.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:28.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:28.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:28.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:28 smithi175 ceph-mon[101434]: pgmap v630: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 23453/210447 objects degraded (11.144%); 2823/210447 objects misplaced (1.341%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:28:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:28 smithi175 ceph-mon[101434]: osdmap e207: 8 total, 7 up, 7 in 2024-08-17T17:28:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23453/210447 objects degraded (11.144%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:28.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:28.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:28.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:28.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[104516]: pgmap v630: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 23453/210447 objects degraded (11.144%); 2823/210447 objects misplaced (1.341%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:28:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[104516]: osdmap e207: 8 total, 7 up, 7 in 2024-08-17T17:28:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23453/210447 objects degraded (11.144%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[109246]: pgmap v630: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 23453/210447 objects degraded (11.144%); 2823/210447 objects misplaced (1.341%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:28:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:28:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[109246]: osdmap e207: 8 total, 7 up, 7 in 2024-08-17T17:28:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23453/210447 objects degraded (11.144%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:28.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:28.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:28.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:28.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:28.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:28.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:29.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:29.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:29.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:29.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:29.410 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:29.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:29.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:29.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:29 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:28:29.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:29.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:29 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:28:29.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:29 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:28:29.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:29.909 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:29.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:30.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:30.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:30.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:30.283 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:30 smithi175 ceph-mon[101434]: pgmap v632: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 23098/206961 objects degraded (11.161%); 2286/206961 objects misplaced (1.105%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:30.283 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:30 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:28:30.283 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:30 smithi175 ceph-mon[101434]: osdmap e208: 8 total, 7 up, 7 in 2024-08-17T17:28:30.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:30.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:30.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:30.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:30.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:30.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[104516]: pgmap v632: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 23098/206961 objects degraded (11.161%); 2286/206961 objects misplaced (1.105%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:28:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[104516]: osdmap e208: 8 total, 7 up, 7 in 2024-08-17T17:28:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[109246]: pgmap v632: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 57 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 23098/206961 objects degraded (11.161%); 2286/206961 objects misplaced (1.105%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:28:30.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:30 smithi064 ceph-mon[109246]: osdmap e208: 8 total, 7 up, 7 in 2024-08-17T17:28:30.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:30.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.824 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:30.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:31.082 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:31.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:31.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:31.310 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:31.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:31.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:31.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:31.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:31.649 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: osdmap e209: 8 total, 7 up, 7 in 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: 6.f scrub starts 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: osdmap e209: 8 total, 7 up, 7 in 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: 6.f scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: osdmap e209: 8 total, 7 up, 7 in 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: 6.f scrub starts 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:31.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.793 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:31.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:31.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:32.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:32.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:32.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:32.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:32.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:32.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:32.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:32.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:32.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:32.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[104516]: pgmap v635: 109 pgs: 2 peering, 10 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 14 active+undersized+degraded, 56 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20226/195255 objects degraded (10.359%); 1686/195255 objects misplaced (0.863%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[104516]: 6.16 scrub starts 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[104516]: osdmap e210: 8 total, 7 up, 7 in 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[104516]: 6.20 scrub starts 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[104516]: 6.1f scrub starts 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[109246]: pgmap v635: 109 pgs: 2 peering, 10 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 14 active+undersized+degraded, 56 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20226/195255 objects degraded (10.359%); 1686/195255 objects misplaced (0.863%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:32.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[109246]: 6.16 scrub starts 2024-08-17T17:28:32.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[109246]: osdmap e210: 8 total, 7 up, 7 in 2024-08-17T17:28:32.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[109246]: 6.20 scrub starts 2024-08-17T17:28:32.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:32 smithi064 ceph-mon[109246]: 6.1f scrub starts 2024-08-17T17:28:32.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:32 smithi175 ceph-mon[101434]: pgmap v635: 109 pgs: 2 peering, 10 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 14 active+undersized+degraded, 56 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20226/195255 objects degraded (10.359%); 1686/195255 objects misplaced (0.863%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:32 smithi175 ceph-mon[101434]: 6.16 scrub starts 2024-08-17T17:28:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:32 smithi175 ceph-mon[101434]: osdmap e210: 8 total, 7 up, 7 in 2024-08-17T17:28:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:32 smithi175 ceph-mon[101434]: 6.20 scrub starts 2024-08-17T17:28:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:32 smithi175 ceph-mon[101434]: 6.1f scrub starts 2024-08-17T17:28:32.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:32.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:32.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:32.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:32.922 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:33.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:33.046 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:33.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:33.090 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:28:33.090 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:33.126 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:28:33.126 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:33.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:33.166 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:28:33.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:33.195 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:33.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:28:33.204 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:33.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:28:33.238 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:33.269 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:28:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:33.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:28:33.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:33.302 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:28:33.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:33.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:28:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/870879887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[104516]: 6.14 scrub starts 2024-08-17T17:28:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 20226/195255 objects degraded (10.359%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:28:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/870879887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[109246]: 6.14 scrub starts 2024-08-17T17:28:33.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 20226/195255 objects degraded (10.359%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:33.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:33.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:33.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:33 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:28:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/870879887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:33 smithi175 ceph-mon[101434]: 6.14 scrub starts 2024-08-17T17:28:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 20226/195255 objects degraded (10.359%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:28:33.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:33.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:33.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:33.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:33.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:33.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:34.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:34.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:34.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:34.212 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:33] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-08-17T17:28:34.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:34.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:34.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:34.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:34.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:34.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:34.518 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:34 smithi064 ceph-mon[104516]: 6.d scrub starts 2024-08-17T17:28:34.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:34 smithi064 ceph-mon[104516]: pgmap v637: 109 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 58 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19901/192474 objects degraded (10.340%); 1686/192474 objects misplaced (0.876%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:28:34.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:34 smithi064 ceph-mon[109246]: 6.d scrub starts 2024-08-17T17:28:34.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:34 smithi064 ceph-mon[109246]: pgmap v637: 109 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 58 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19901/192474 objects degraded (10.340%); 1686/192474 objects misplaced (0.876%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:28:34.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:34.649 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:34.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:34 smithi175 ceph-mon[101434]: 6.d scrub starts 2024-08-17T17:28:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:34 smithi175 ceph-mon[101434]: pgmap v637: 109 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 24 active+undersized, 15 active+undersized+degraded, 58 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19901/192474 objects degraded (10.340%); 1686/192474 objects misplaced (0.876%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:28:34.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:34.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:34.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:34.897 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:34.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:35.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:35.040 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:35.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:35.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:35.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:35.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:35.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:35.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:35.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:35.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:35.546 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:35.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[104516]: 6.1d scrub starts 2024-08-17T17:28:35.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[104516]: 6.b scrub starts 2024-08-17T17:28:35.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[104516]: osdmap e211: 8 total, 7 up, 7 in 2024-08-17T17:28:35.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[109246]: 6.1d scrub starts 2024-08-17T17:28:35.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[109246]: 6.b scrub starts 2024-08-17T17:28:35.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:35 smithi064 ceph-mon[109246]: osdmap e211: 8 total, 7 up, 7 in 2024-08-17T17:28:35.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:35.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:35 smithi175 ceph-mon[101434]: 6.1d scrub starts 2024-08-17T17:28:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:35 smithi175 ceph-mon[101434]: 6.b scrub starts 2024-08-17T17:28:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:35 smithi175 ceph-mon[101434]: osdmap e211: 8 total, 7 up, 7 in 2024-08-17T17:28:35.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:35.796 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:35.818 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:35.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:35.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.064 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:36.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:36.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:36.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:36.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[104516]: pgmap v639: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 795 op/s; 21654/190038 objects degraded (11.395%); 84/190038 objects misplaced (0.044%); 11 MiB/s, 178 objects/s recovering 2024-08-17T17:28:36.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[104516]: osdmap e212: 8 total, 7 up, 7 in 2024-08-17T17:28:36.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:28:36.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[109246]: pgmap v639: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 795 op/s; 21654/190038 objects degraded (11.395%); 84/190038 objects misplaced (0.044%); 11 MiB/s, 178 objects/s recovering 2024-08-17T17:28:36.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[109246]: osdmap e212: 8 total, 7 up, 7 in 2024-08-17T17:28:36.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:36 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:28:36.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:36.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:36.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:36.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:36 smithi175 ceph-mon[101434]: pgmap v639: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 795 op/s; 21654/190038 objects degraded (11.395%); 84/190038 objects misplaced (0.044%); 11 MiB/s, 178 objects/s recovering 2024-08-17T17:28:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:36 smithi175 ceph-mon[101434]: osdmap e212: 8 total, 7 up, 7 in 2024-08-17T17:28:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:36 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:28:36.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:36.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:36.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:36.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:36.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:36.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.018 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:28:37.018 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:28:37.018 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:28:37.018 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.273512 2024-08-17T17:28:37.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 0.27351248407744877 2024-08-17T17:28:37.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:37.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:37.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:37.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:28:37.254 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:28:37.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:37.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:37.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:37.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:37.510 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:37.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:37.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:37 smithi175 ceph-mon[101434]: 6.1b scrub starts 2024-08-17T17:28:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:37 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:28:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:37 smithi175 ceph-mon[101434]: 6.f scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[104516]: 6.1b scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[104516]: 6.f scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[109246]: 6.1b scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:28:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:37 smithi064 ceph-mon[109246]: 6.f scrub starts 2024-08-17T17:28:37.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:37.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:37.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:37.920 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:38.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:38.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:38.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:38.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:38.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[104516]: pgmap v641: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 1.65k op/s; 21184/185745 objects degraded (11.405%); 84/185745 objects misplaced (0.045%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[104516]: 6.20 scrub starts 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[104516]: 6.1f scrub starts 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 21184/185745 objects degraded (11.405%), 21 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[109246]: pgmap v641: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 1.65k op/s; 21184/185745 objects degraded (11.405%); 84/185745 objects misplaced (0.045%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:28:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:28:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[109246]: 6.20 scrub starts 2024-08-17T17:28:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[109246]: 6.1f scrub starts 2024-08-17T17:28:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 21184/185745 objects degraded (11.405%), 21 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:38.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.531 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:38.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:38 smithi175 ceph-mon[101434]: pgmap v641: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 1.65k op/s; 21184/185745 objects degraded (11.405%); 84/185745 objects misplaced (0.045%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:28:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:38 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:28:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:38 smithi175 ceph-mon[101434]: 6.20 scrub starts 2024-08-17T17:28:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:38 smithi175 ceph-mon[101434]: 6.1f scrub starts 2024-08-17T17:28:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 21184/185745 objects degraded (11.405%), 21 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:38.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:38.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:38.804 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:38.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:38.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:38.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:39.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:39.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:39.123 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:39.124 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:39.144 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:39.158 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:39.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:39.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:39.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:39.350 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:39.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:39.373 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:28:39.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:39.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:39.593 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:39.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:39.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:39 smithi175 ceph-mon[101434]: 6.d scrub starts 2024-08-17T17:28:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:39 smithi175 ceph-mon[101434]: 6.16 scrub starts 2024-08-17T17:28:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:39 smithi175 ceph-mon[101434]: 6.14 scrub starts 2024-08-17T17:28:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:39 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/310924588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:39 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[104516]: 6.d scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[104516]: 6.16 scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[104516]: 6.14 scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/310924588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[109246]: 6.d scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[109246]: 6.16 scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[109246]: 6.14 scrub starts 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/310924588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:39 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:39.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:39.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:39.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:39.883 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:39.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:40.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:40.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:40.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:40.261 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:40.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:40.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:40.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:40.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:40.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:40.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: 6.1d scrub starts 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: pgmap v642: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 134 B/s rd, 28 MiB/s wr, 1.42k op/s; 20664/181116 objects degraded (11.409%); 84/181116 objects misplaced (0.046%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: osdmap e213: 8 total, 7 up, 7 in 2024-08-17T17:28:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: 6.1d scrub starts 2024-08-17T17:28:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: pgmap v642: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 134 B/s rd, 28 MiB/s wr, 1.42k op/s; 20664/181116 objects degraded (11.409%); 84/181116 objects misplaced (0.046%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:28:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:28:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: osdmap e213: 8 total, 7 up, 7 in 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: 6.1d scrub starts 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: pgmap v642: 109 pgs: 1 active+recovering+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 134 B/s rd, 28 MiB/s wr, 1.42k op/s; 20664/181116 objects degraded (11.409%); 84/181116 objects misplaced (0.046%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: osdmap e213: 8 total, 7 up, 7 in 2024-08-17T17:28:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:40.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:40.777 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:40.879 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:41.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:41.028 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:41.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:41.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:41.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:41.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:41.289 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:41.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:41.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:41.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:41.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:41.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:41.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:41 smithi175 ceph-mon[101434]: 6.9 scrub starts 2024-08-17T17:28:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:41 smithi064 ceph-mon[104516]: 6.9 scrub starts 2024-08-17T17:28:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:41 smithi064 ceph-mon[109246]: 6.9 scrub starts 2024-08-17T17:28:41.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:41.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:41.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:41.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:41.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:42.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:42.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:42.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:42.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:28:42.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:42.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:42.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:42.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: pgmap v644: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 154 B/s rd, 32 MiB/s wr, 1.60k op/s; 18053/173949 objects degraded (10.378%); 1731/173949 objects misplaced (0.995%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.b scrub starts 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.9 scrub ok 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.8 scrub starts 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.0 scrub starts 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.b scrub ok 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.8 scrub ok 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.0 scrub ok 2024-08-17T17:28:42.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[109246]: 6.f scrub starts 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: pgmap v644: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 154 B/s rd, 32 MiB/s wr, 1.60k op/s; 18053/173949 objects degraded (10.378%); 1731/173949 objects misplaced (0.995%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.b scrub starts 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.9 scrub ok 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.8 scrub starts 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.0 scrub starts 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.b scrub ok 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.8 scrub ok 2024-08-17T17:28:42.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.0 scrub ok 2024-08-17T17:28:42.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:42 smithi064 ceph-mon[104516]: 6.f scrub starts 2024-08-17T17:28:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:42.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:42.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:42.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: pgmap v644: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 154 B/s rd, 32 MiB/s wr, 1.60k op/s; 18053/173949 objects degraded (10.378%); 1731/173949 objects misplaced (0.995%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.b scrub starts 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.9 scrub ok 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.8 scrub starts 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.0 scrub starts 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.b scrub ok 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.8 scrub ok 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.0 scrub ok 2024-08-17T17:28:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:42 smithi175 ceph-mon[101434]: 6.f scrub starts 2024-08-17T17:28:42.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.813 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:42.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:42.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:42.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:43.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:43.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:43.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:43.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:43.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:43.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:43.604 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.2 scrub starts 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.2 scrub ok 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.18 scrub starts 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.20 scrub starts 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.f scrub ok 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.18 scrub ok 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.20 scrub ok 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 18053/173949 objects degraded (10.378%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:43 smithi175 ceph-mon[101434]: 6.1f scrub starts 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.2 scrub starts 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.2 scrub ok 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.18 scrub starts 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.20 scrub starts 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.f scrub ok 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.18 scrub ok 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.20 scrub ok 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 18053/173949 objects degraded (10.378%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[104516]: 6.1f scrub starts 2024-08-17T17:28:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.2 scrub starts 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.2 scrub ok 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.18 scrub starts 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.20 scrub starts 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.f scrub ok 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.18 scrub ok 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.20 scrub ok 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 18053/173949 objects degraded (10.378%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:43.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-mon[109246]: 6.1f scrub starts 2024-08-17T17:28:43.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:28:43.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:43.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:43.977 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:44.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:44.049 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:43] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-08-17T17:28:44.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:44.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:44.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:44.285 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:44.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:44.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:44.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:44.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: pgmap v645: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 17483/168747 objects degraded (10.360%); 1731/168747 objects misplaced (1.026%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.1b scrub starts 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.d scrub starts 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.1b scrub ok 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.d scrub ok 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.1f scrub ok 2024-08-17T17:28:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:44 smithi175 ceph-mon[101434]: 6.14 scrub starts 2024-08-17T17:28:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: pgmap v645: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 17483/168747 objects degraded (10.360%); 1731/168747 objects misplaced (1.026%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.1b scrub starts 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.d scrub starts 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.1b scrub ok 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.d scrub ok 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.1f scrub ok 2024-08-17T17:28:44.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[104516]: 6.14 scrub starts 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: pgmap v645: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 17483/168747 objects degraded (10.360%); 1731/168747 objects misplaced (1.026%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.1b scrub starts 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.d scrub starts 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.1b scrub ok 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.d scrub ok 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.1f scrub ok 2024-08-17T17:28:44.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:44 smithi064 ceph-mon[109246]: 6.14 scrub starts 2024-08-17T17:28:44.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:44.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:44.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:44.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:44.830 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:44.861 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:44.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:44.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:45.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:45.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:45.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:45.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:45.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:45.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:45.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:45.419 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 0.273512 (8179242) 2024-08-17T17:28:45.419 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:45.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:45.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:45.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: 6.6 scrub starts 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: 6.1d scrub starts 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: 6.6 scrub ok 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: 6.1d scrub ok 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: 6.14 scrub ok 2024-08-17T17:28:45.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877} : dispatch 2024-08-17T17:28:45.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: 6.6 scrub starts 2024-08-17T17:28:45.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: 6.1d scrub starts 2024-08-17T17:28:45.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: 6.6 scrub ok 2024-08-17T17:28:45.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: 6.1d scrub ok 2024-08-17T17:28:45.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: 6.14 scrub ok 2024-08-17T17:28:45.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877} : dispatch 2024-08-17T17:28:45.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:45.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: 6.6 scrub starts 2024-08-17T17:28:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: 6.1d scrub starts 2024-08-17T17:28:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: 6.6 scrub ok 2024-08-17T17:28:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: 6.1d scrub ok 2024-08-17T17:28:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: 6.14 scrub ok 2024-08-17T17:28:45.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877} : dispatch 2024-08-17T17:28:45.683 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:45.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:45.785 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:45.810 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:45.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:45.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:46.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:46.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:46.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:46.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:46.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:46.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:46.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:46.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:46.620 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: pgmap v646: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16234/156138 objects degraded (10.397%); 1555/156138 objects misplaced (0.996%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: 6.e scrub starts 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: 6.16 scrub starts 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: 6.e scrub ok 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877}]': finished 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: osdmap e214: 8 total, 7 up, 7 in 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[104516]: 6.16 scrub ok 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: pgmap v646: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16234/156138 objects degraded (10.397%); 1555/156138 objects misplaced (0.996%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: 6.e scrub starts 2024-08-17T17:28:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: 6.16 scrub starts 2024-08-17T17:28:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: 6.e scrub ok 2024-08-17T17:28:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877}]': finished 2024-08-17T17:28:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: osdmap e214: 8 total, 7 up, 7 in 2024-08-17T17:28:46.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:46 smithi064 ceph-mon[109246]: 6.16 scrub ok 2024-08-17T17:28:46.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:46.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:46.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:46.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:46.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: pgmap v646: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16234/156138 objects degraded (10.397%); 1555/156138 objects misplaced (0.996%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: 6.e scrub starts 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: 6.16 scrub starts 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: 6.e scrub ok 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4231203708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.27351248407744877}]': finished 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: osdmap e214: 8 total, 7 up, 7 in 2024-08-17T17:28:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:46 smithi175 ceph-mon[101434]: 6.16 scrub ok 2024-08-17T17:28:46.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:47.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:47.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:47.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:47.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:47.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:47.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:47.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:47.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:47.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:47.660 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:47.684 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[104516]: 6.17 scrub starts 2024-08-17T17:28:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[104516]: 6.10 scrub starts 2024-08-17T17:28:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[104516]: 6.17 scrub ok 2024-08-17T17:28:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[109246]: 6.17 scrub starts 2024-08-17T17:28:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[109246]: 6.10 scrub starts 2024-08-17T17:28:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:47 smithi064 ceph-mon[109246]: 6.17 scrub ok 2024-08-17T17:28:47.786 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:47.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:47 smithi175 ceph-mon[101434]: 6.17 scrub starts 2024-08-17T17:28:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:47 smithi175 ceph-mon[101434]: 6.10 scrub starts 2024-08-17T17:28:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:47 smithi175 ceph-mon[101434]: 6.17 scrub ok 2024-08-17T17:28:47.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:47.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:48.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:48.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:48.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:48.621 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:48.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[104516]: pgmap v648: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 15689/151452 objects degraded (10.359%); 1555/151452 objects misplaced (1.027%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[104516]: 6.10 scrub ok 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 15689/151452 objects degraded (10.359%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[109246]: pgmap v648: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 15689/151452 objects degraded (10.359%); 1555/151452 objects misplaced (1.027%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[109246]: 6.10 scrub ok 2024-08-17T17:28:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 15689/151452 objects degraded (10.359%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:48.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:48.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:48.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:48.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:48 smithi175 ceph-mon[101434]: pgmap v648: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 15689/151452 objects degraded (10.359%); 1555/151452 objects misplaced (1.027%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:28:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:48 smithi175 ceph-mon[101434]: 6.10 scrub ok 2024-08-17T17:28:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 15689/151452 objects degraded (10.359%), 20 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:49.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:49.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:49.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:49.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:49.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:49.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:49.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:49.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:49.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:49.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:49.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:49.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:49.648 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[104516]: 6.13 scrub starts 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[104516]: 6.12 scrub starts 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[104516]: 6.12 scrub ok 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[104516]: 6.13 scrub ok 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[109246]: 6.13 scrub starts 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[109246]: 6.12 scrub starts 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[109246]: 6.12 scrub ok 2024-08-17T17:28:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:49 smithi064 ceph-mon[109246]: 6.13 scrub ok 2024-08-17T17:28:49.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:49.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:49.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:49.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:49 smithi175 ceph-mon[101434]: 6.13 scrub starts 2024-08-17T17:28:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:49 smithi175 ceph-mon[101434]: 6.12 scrub starts 2024-08-17T17:28:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:49 smithi175 ceph-mon[101434]: 6.12 scrub ok 2024-08-17T17:28:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:49 smithi175 ceph-mon[101434]: 6.13 scrub ok 2024-08-17T17:28:49.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:50.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:50.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:50.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:50.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:50.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:50.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:50.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:50.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:50.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:50.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:50.692 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:50.741 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:50 smithi064 ceph-mon[104516]: pgmap v649: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 63 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 13768/146943 objects degraded (9.370%); 2998/146943 objects misplaced (2.040%) 2024-08-17T17:28:50.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:50 smithi064 ceph-mon[109246]: pgmap v649: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 63 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 13768/146943 objects degraded (9.370%); 2998/146943 objects misplaced (2.040%) 2024-08-17T17:28:50.794 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:50.818 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:50 smithi175 ceph-mon[101434]: pgmap v649: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 24 active+undersized, 19 active+undersized+degraded, 63 active+clean; 3.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 13768/146943 objects degraded (9.370%); 2998/146943 objects misplaced (2.040%) 2024-08-17T17:28:50.920 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:50.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:51.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:51.087 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:51.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:51.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:51.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:51.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:51.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:51.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:51.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:51.654 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:51.756 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:51.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:51.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:51.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:52.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:52.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:52.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.196 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.298 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:52.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:52.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[104516]: pgmap v650: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 13049/139521 objects degraded (9.353%); 2869/139521 objects misplaced (2.056%) 2024-08-17T17:28:52.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[104516]: 6.22 scrub starts 2024-08-17T17:28:52.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[104516]: 6.22 scrub ok 2024-08-17T17:28:52.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[109246]: pgmap v650: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 13049/139521 objects degraded (9.353%); 2869/139521 objects misplaced (2.056%) 2024-08-17T17:28:52.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[109246]: 6.22 scrub starts 2024-08-17T17:28:52.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:52 smithi064 ceph-mon[109246]: 6.22 scrub ok 2024-08-17T17:28:52.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:52.706 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:52.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:52 smithi175 ceph-mon[101434]: pgmap v650: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 13049/139521 objects degraded (9.353%); 2869/139521 objects misplaced (2.056%) 2024-08-17T17:28:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:52 smithi175 ceph-mon[101434]: 6.22 scrub starts 2024-08-17T17:28:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:52 smithi175 ceph-mon[101434]: 6.22 scrub ok 2024-08-17T17:28:52.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:53.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:53.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:53.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:53.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:53.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:53.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:53.335 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:53.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:53.474 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:53.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:53.615 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:53.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:53.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 13049/139521 objects degraded (9.353%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 13049/139521 objects degraded (9.353%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:53.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:53.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:53.858 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:53.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:53.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 13049/139521 objects degraded (9.353%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:53.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:54.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:54.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:54.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:54.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:54.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:54.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:54.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:54.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:28:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:28:53] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-08-17T17:28:54.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:54.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:54.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:54.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:54.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:54.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:54 smithi175 ceph-mon[101434]: pgmap v651: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12442/134136 objects degraded (9.276%); 2869/134136 objects misplaced (2.139%) 2024-08-17T17:28:54.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:54 smithi064 ceph-mon[104516]: pgmap v651: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12442/134136 objects degraded (9.276%); 2869/134136 objects misplaced (2.139%) 2024-08-17T17:28:54.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:54 smithi064 ceph-mon[109246]: pgmap v651: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12442/134136 objects degraded (9.276%); 2869/134136 objects misplaced (2.139%) 2024-08-17T17:28:54.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:55.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:55.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:55.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:55.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:55.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:55.359 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:55.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:55.476 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:28:55.476 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:28:55.476 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-08-17T17:28:55.476 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:28:55.476 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:28:55.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-08-17T17:28:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:55.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:55.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.637 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:55.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:55.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:55.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:55.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:55.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:55.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:56.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:56.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:56.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:56.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:56.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:56.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:56.434 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:56.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:56.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:56.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:56.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:56.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:56.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:56 smithi175 ceph-mon[101434]: pgmap v652: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11261/120414 objects degraded (9.352%); 2430/120414 objects misplaced (2.018%) 2024-08-17T17:28:56.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:56 smithi175 ceph-mon[101434]: 6.1e scrub starts 2024-08-17T17:28:56.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:56 smithi175 ceph-mon[101434]: 6.1e scrub ok 2024-08-17T17:28:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:56.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:56.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[104516]: pgmap v652: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11261/120414 objects degraded (9.352%); 2430/120414 objects misplaced (2.018%) 2024-08-17T17:28:56.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[104516]: 6.1e scrub starts 2024-08-17T17:28:56.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[104516]: 6.1e scrub ok 2024-08-17T17:28:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[109246]: pgmap v652: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11261/120414 objects degraded (9.352%); 2430/120414 objects misplaced (2.018%) 2024-08-17T17:28:56.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[109246]: 6.1e scrub starts 2024-08-17T17:28:56.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:56 smithi064 ceph-mon[109246]: 6.1e scrub ok 2024-08-17T17:28:56.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:56.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:57.099 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:57.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.122 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:57.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:57.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:57.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:57.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:57.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:57.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:57.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:57.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:57.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:57.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:57.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:57.809 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:57.810 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:57.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:57.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:58.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:58.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:58.131 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:58.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[104516]: pgmap v653: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10734/115671 objects degraded (9.280%); 2430/115671 objects misplaced (2.101%) 2024-08-17T17:28:58.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[109246]: pgmap v653: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10734/115671 objects degraded (9.280%); 2430/115671 objects misplaced (2.101%) 2024-08-17T17:28:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:58.288 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:58.390 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:58.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:57 smithi175 ceph-mon[101434]: pgmap v653: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10734/115671 objects degraded (9.280%); 2430/115671 objects misplaced (2.101%) 2024-08-17T17:28:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:58.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:58.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:58.542 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:58.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:58.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:58.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.793 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:58.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:58.937 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:59.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:59.063 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:59.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 10734/115671 objects degraded (9.280%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 10734/115671 objects degraded (9.280%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:59.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:28:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:59.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:59.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:59.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 10734/115671 objects degraded (9.280%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:28:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:28:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:28:59.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:59.474 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:59.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:59.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:28:59.622 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:59.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:59.762 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:28:59.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:59.842 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:28:59.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:59.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:28:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:59.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:00.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:00.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:00.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:29:00.121 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:00.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:29:00.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:00.164 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.193 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:29:00.193 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:00.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:00.228 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:00 smithi175 ceph-mon[101434]: pgmap v654: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10421/110925 objects degraded (9.395%); 2258/110925 objects misplaced (2.036%) 2024-08-17T17:29:00.228 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2260775900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.231 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:00.238 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:29:00.238 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:00.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:00.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.318 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:00.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:00.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:00 smithi064 ceph-mon[104516]: pgmap v654: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10421/110925 objects degraded (9.395%); 2258/110925 objects misplaced (2.036%) 2024-08-17T17:29:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2260775900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:00 smithi064 ceph-mon[109246]: pgmap v654: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10421/110925 objects degraded (9.395%); 2258/110925 objects misplaced (2.036%) 2024-08-17T17:29:00.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2260775900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:00.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.617 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:00.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:00.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:00.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:00.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:01.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:29:01.040 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:29:01.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:01.157 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:01.197 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:29:01.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:01.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:01.320 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:01.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:01.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:01.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:01.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:01.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:01.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:01.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:01.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:01.720 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:29:01.720 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:29:01.720 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:185114 6.e 6:73cc20d4:::benchmark_data_smithi064_185391_object77381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:29:01.707073+0000", 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.01219685, 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.012219170999999999, 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 185114 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.721 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707073+0000", 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707073+0000", 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:01.722 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707072+0000", 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707078+0000", 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1850000000000001e-06 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707088+0000", 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0594999999999999e-05 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707093+0000", 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1290000000000004e-06 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707133+0000", 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.943e-05 2024-08-17T17:29:01.724 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707251+0000", 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000117704 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.707682+0000", 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00043151799999999998 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.725 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.709433+0000", 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001750751 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.710642+0000", 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0012093430000000001 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.726 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35007.0:185125 6.12 e215/194 6:4d6a65c3:::benchmark_data_smithi064_185391_object77392:head v 215'7367, mlcod=215'7365)", 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:29:01.718005+0000", 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.001264996, 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001339392, 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718005+0000", 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.727 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718005+0000", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718004+0000", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718011+0000", 2024-08-17T17:29:01.728 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0360000000000001e-06 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718025+0000", 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3597e-05 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718032+0000", 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2960000000000001e-06 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.729 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718077+0000", 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5098000000000001e-05 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718096+0000", 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8872000000000002e-05 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.730 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35007.0:185126 6.15 e215/200 6:af863a60:::benchmark_data_smithi064_185391_object77393:head v 215'7444, mlcod=215'7442)", 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:29:01.718271+0000", 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.00099909999999999994, 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001106704, 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718271+0000", 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.731 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718271+0000", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718270+0000", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718276+0000", 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4670000000000003e-06 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.732 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718288+0000", 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1989e-05 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718294+0000", 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5309999999999998e-06 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:01.733 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718354+0000", 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9837999999999999e-05 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:29:01.718375+0000", 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.086e-05 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 3 2024-08-17T17:29:01.734 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:29:01.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:01.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:01.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:01.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:02.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:02.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:02.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:02.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:02.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:02.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[104516]: pgmap v655: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9720/103524 objects degraded (9.389%); 2092/103524 objects misplaced (2.021%) 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[104516]: osdmap e215: 8 total, 7 up, 7 in 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[109246]: pgmap v655: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9720/103524 objects degraded (9.389%); 2092/103524 objects misplaced (2.021%) 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:02 smithi064 ceph-mon[109246]: osdmap e215: 8 total, 7 up, 7 in 2024-08-17T17:29:02.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:02.535 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:02.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:02 smithi175 ceph-mon[101434]: pgmap v655: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9720/103524 objects degraded (9.389%); 2092/103524 objects misplaced (2.021%) 2024-08-17T17:29:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:02 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2927974392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:02 smithi175 ceph-mon[101434]: osdmap e215: 8 total, 7 up, 7 in 2024-08-17T17:29:02.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:02.778 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:02.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:02.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:02.930 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:03.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:03.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:03.070 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:03.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:03.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:03.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:03.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:03.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9720/103524 objects degraded (9.389%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:03.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9720/103524 objects degraded (9.389%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:03.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:03.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:03.602 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9720/103524 objects degraded (9.389%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:03.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:03.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:03.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.869 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:03.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.997 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:03.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:04.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:04.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:04.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:04.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:04.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:04.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:29:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:04 smithi064 ceph-mon[104516]: pgmap v657: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9171/98649 objects degraded (9.297%); 2092/98649 objects misplaced (2.121%) 2024-08-17T17:29:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:04 smithi064 ceph-mon[109246]: pgmap v657: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9171/98649 objects degraded (9.297%); 2092/98649 objects misplaced (2.121%) 2024-08-17T17:29:04.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:03] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-08-17T17:29:04.467 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:04 smithi175 ceph-mon[101434]: pgmap v657: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9171/98649 objects degraded (9.297%); 2092/98649 objects misplaced (2.121%) 2024-08-17T17:29:04.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:04.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:04.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:29:04.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:04.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.682 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:04.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.786 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:04.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:04.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:04.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:05.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:05.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:05.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:05.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.360 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:05.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:05.461 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:05.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:05.484 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:05.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:05.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:05.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:05.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:05.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:05.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:06.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:06.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:06.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:06.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:06.215 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:06.262 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:29:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:29:06.262 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:29:06.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:06 smithi064 ceph-mon[104516]: pgmap v658: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 8159/86265 objects degraded (9.458%); 1700/86265 objects misplaced (1.971%) 2024-08-17T17:29:06.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:06 smithi064 ceph-mon[109246]: pgmap v658: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 8159/86265 objects degraded (9.458%); 1700/86265 objects misplaced (1.971%) 2024-08-17T17:29:06.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.313 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.414 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:06.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:06.456 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:06 smithi175 ceph-mon[101434]: pgmap v658: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 8159/86265 objects degraded (9.458%); 1700/86265 objects misplaced (1.971%) 2024-08-17T17:29:06.466 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:29:06.466 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:29:06.466 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:29:06.466 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:29:06.466 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:29:06.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:06.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:06.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:06.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:06.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:06.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:06.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:07.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:07.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:29:07.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:07.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:07.288 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:07.377 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:07.390 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:07.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:07.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:07.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:07.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:07.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:07.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:07.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:07.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:07.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:07.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:07.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:07.997 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:08.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:08.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:08.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:08.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:08.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:08.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:08.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:08 smithi064 ceph-mon[104516]: pgmap v659: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7643/81633 objects degraded (9.363%); 1700/81633 objects misplaced (2.082%) 2024-08-17T17:29:08.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 7643/81633 objects degraded (9.363%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:08.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:08 smithi064 ceph-mon[109246]: pgmap v659: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7643/81633 objects degraded (9.363%); 1700/81633 objects misplaced (2.082%) 2024-08-17T17:29:08.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 7643/81633 objects degraded (9.363%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:08.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:08.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:08.644 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:08 smithi175 ceph-mon[101434]: pgmap v659: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7643/81633 objects degraded (9.363%); 1700/81633 objects misplaced (2.082%) 2024-08-17T17:29:08.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 7643/81633 objects degraded (9.363%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:08.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:08.716 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:08.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:08.859 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:08.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:08.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:09.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:09.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140794 5.e 5:70adc07f:::benchmark_data_smithi064_147470_object8850:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.29968191199998, 2024-08-17T17:29:09.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.01103878, 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140794 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217389+0000", 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217387+0000", 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217394+0000", 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8109999999999997e-06 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217408+0000", 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3536999999999999e-05 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217415+0000", 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5680000000000002e-06 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217613+0000", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019751099999999999 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.217621+0000", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4849999999999992e-06 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210300+0000", 2024-08-17T17:29:09.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.99267841499999998 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210423+0000", 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012385 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210935+0000", 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00051183100000000003 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225429+0000", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014493239999999999 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225526+0000", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7489000000000001e-05 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228353+0000", 2024-08-17T17:29:09.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0028269509999999999 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228388+0000", 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5176000000000003e-05 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228428+0000", 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9483999999999997e-05 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.527 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140802 5.1 5:8420ad0a:::benchmark_data_smithi064_147470_object8858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.234059876, 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93395603000000005, 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140802 2024-08-17T17:29:09.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283011+0000", 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283009+0000", 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283017+0000", 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6709999999999999e-06 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283032+0000", 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5159e-05 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283063+0000", 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0406e-05 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283108+0000", 2024-08-17T17:29:09.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4935999999999998e-05 2024-08-17T17:29:09.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.283118+0000", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0757000000000001e-05 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210288+0000", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.92716919799999997 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210421+0000", 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013376499999999999 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.210968+0000", 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054626499999999999 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216852+0000", 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058840200000000002 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216887+0000", 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5522000000000003e-05 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216894+0000", 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.63e-06 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216933+0000", 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9208999999999997e-05 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216967+0000", 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3924000000000002e-05 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140822 5.1 5:8accaffa:::benchmark_data_smithi064_147470_object8878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.13979115799998, 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85167474700000001, 2024-08-17T17:29:09.537 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140822 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377280+0000", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377278+0000", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377285+0000", 2024-08-17T17:29:09.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377301+0000", 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5423000000000001e-05 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377309+0000", 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0390000000000005e-06 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377380+0000", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0784999999999994e-05 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377389+0000", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1179999999999998e-06 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.216990+0000", 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83960149699999997 2024-08-17T17:29:09.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217105+0000", 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114775 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.217195+0000", 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9908999999999996e-05 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228788+0000", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011592966 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228835+0000", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6978000000000003e-05 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228873+0000", 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7605000000000003e-05 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228921+0000", 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8013000000000003e-05 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228954+0000", 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3775000000000001e-05 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140827 5.1 5:8b17e64e:::benchmark_data_smithi064_147470_object8883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.13974491200003, 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85178876800000003, 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140827 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377326+0000", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377325+0000", 2024-08-17T17:29:09.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377328+0000", 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1370000000000002e-06 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377335+0000", 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8260000000000004e-06 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377340+0000", 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1730000000000003e-06 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377409+0000", 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8702999999999996e-05 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.377415+0000", 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8869999999999997e-06 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220644+0000", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84322923900000002 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220744+0000", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9375000000000003e-05 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.220857+0000", 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000113417 2024-08-17T17:29:09.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228842+0000", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079852320000000001 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228994+0000", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001518 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229024+0000", 2024-08-17T17:29:09.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0190999999999999e-05 2024-08-17T17:29:09.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229078+0000", 2024-08-17T17:29:09.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3427999999999997e-05 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229115+0000", 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6875999999999997e-05 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.553 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140835 5.1 5:88614440:::benchmark_data_smithi064_147470_object8891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.13157842300001, 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85468788299999998, 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.554 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140835 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385492+0000", 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385491+0000", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385500+0000", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.738e-06 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385518+0000", 2024-08-17T17:29:09.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8902e-05 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385527+0000", 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7320000000000004e-06 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385583+0000", 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6035000000000001e-05 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.385593+0000", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4089999999999992e-06 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224815+0000", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83922198699999995 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.224935+0000", 2024-08-17T17:29:09.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120415 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225062+0000", 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000127467 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229173+0000", 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041108539999999997 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235012+0000", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0058387229999999997 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240111+0000", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0050989340000000003 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240148+0000", 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6748000000000001e-05 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240180+0000", 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2509999999999999e-05 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140846 5.1 5:8aef7983:::benchmark_data_smithi064_147470_object8902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.11409485600001, 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84633296700000005, 2024-08-17T17:29:09.561 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140846 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402976+0000", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402975+0000", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402981+0000", 2024-08-17T17:29:09.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.19e-06 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.402993+0000", 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1949e-05 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403001+0000", 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403040+0000", 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9366000000000002e-05 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.403047+0000", 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3019999999999997e-06 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229200+0000", 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82615223199999999 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229313+0000", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011332699999999999 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229435+0000", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012217400000000001 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240217+0000", 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010781689000000001 2024-08-17T17:29:09.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.240246+0000", 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9513999999999999e-05 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249221+0000", 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0089744300000000003 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249270+0000", 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9008999999999998e-05 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249309+0000", 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9031e-05 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140847 5.e 5:7ffdcb03:::benchmark_data_smithi064_147470_object8903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.568 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.11231768900001, 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82381208399999994, 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140847 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404753+0000", 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404752+0000", 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:09.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404758+0000", 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7039999999999999e-06 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404770+0000", 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2148e-05 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404777+0000", 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6889999999999998e-06 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404822+0000", 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4827000000000003e-05 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.404854+0000", 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2366999999999997e-05 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218174+0000", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81331985799999995 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218278+0000", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010474799999999999 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.218390+0000", 2024-08-17T17:29:09.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111401 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225460+0000", 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0070701369999999998 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225563+0000", 2024-08-17T17:29:09.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010276 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228497+0000", 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002934613 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228535+0000", 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7539000000000001e-05 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228565+0000", 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0238000000000001e-05 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140853 5.1 5:8ec8c268:::benchmark_data_smithi064_147470_object8909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.11065643900002, 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.843158564, 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.576 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140853 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406414+0000", 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406413+0000", 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406419+0000", 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.057e-06 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406433+0000", 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4328000000000001e-05 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406440+0000", 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8419999999999999e-06 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406489+0000", 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9506999999999999e-05 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.406498+0000", 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3429999999999996e-06 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235037+0000", 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82853932900000005 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235131+0000", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4499000000000004e-05 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.235239+0000", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107617 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249349+0000", 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014109981000000001 2024-08-17T17:29:09.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249391+0000", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1950999999999999e-05 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249499+0000", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010786899999999999 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249541+0000", 2024-08-17T17:29:09.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249573+0000", 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2308999999999998e-05 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140856 5.e 5:7672cac1:::benchmark_data_smithi064_147470_object8912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:29:09.583 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.102820737, 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82466934800000002, 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140856 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:29:09.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414250+0000", 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414249+0000", 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414255+0000", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414267+0000", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1759e-05 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414272+0000", 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2039999999999996e-06 2024-08-17T17:29:09.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414318+0000", 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6096e-05 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.414326+0000", 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222146+0000", 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.80782070699999997 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222239+0000", 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2288999999999997e-05 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.222337+0000", 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8740000000000007e-05 2024-08-17T17:29:09.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.228879+0000", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0065412170000000002 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.230356+0000", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014777900000000001 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238835+0000", 2024-08-17T17:29:09.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0084788959999999997 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238881+0000", 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5371000000000003e-05 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238919+0000", 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8642e-05 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.590 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140876 5.1 5:8e383c61:::benchmark_data_smithi064_147470_object8932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.06581449599997, 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79844427600000001, 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140876 2024-08-17T17:29:09.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451256+0000", 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451255+0000", 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451262+0000", 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1709999999999996e-06 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451275+0000", 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2833000000000001e-05 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451293+0000", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8309999999999999e-05 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451336+0000", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3052e-05 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.451346+0000", 2024-08-17T17:29:09.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8879999999999992e-06 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243525+0000", 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79217860100000004 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243656+0000", 2024-08-17T17:29:09.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013163500000000001 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.243790+0000", 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013325699999999999 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249472+0000", 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0056827220000000003 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249604+0000", 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000131984 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249646+0000", 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2058999999999998e-05 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249675+0000", 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8381000000000002e-05 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.249701+0000", 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5769e-05 2024-08-17T17:29:09.598 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:140877 5.e 5:70e8904e:::benchmark_data_smithi064_147470_object8933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 408.06359729299999, 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.78553511300000001, 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.599 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 140877 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453473+0000", 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453472+0000", 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453480+0000", 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453490+0000", 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0473e-05 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453497+0000", 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0570000000000004e-06 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453537+0000", 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9613000000000002e-05 2024-08-17T17:29:09.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:21.453544+0000", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225585+0000", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.77204120700000001 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225685+0000", 2024-08-17T17:29:09.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000100185 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.225791+0000", 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010613800000000001 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.229184+0000", 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033933909999999999 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238791+0000", 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096060130000000001 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238950+0000", 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015964399999999999 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.238981+0000", 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1239999999999999e-05 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:22.239009+0000", 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7203e-05 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150664 5.f 5:f7ffbecc:::benchmark_data_smithi064_147470_object18720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 391.82296329600001, 2024-08-17T17:29:09.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24416275700000001, 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150664 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694107+0000", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694106+0000", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694113+0000", 2024-08-17T17:29:09.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1339999999999998e-06 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694126+0000", 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2644e-05 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.694132+0000", 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6150000000000002e-06 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.739879+0000", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.045747115999999997 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.740006+0000", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012663900000000001 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.792997+0000", 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.052990644000000003 2024-08-17T17:29:09.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793094+0000", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7040000000000006e-05 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.793282+0000", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018835799999999999 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804038+0000", 2024-08-17T17:29:09.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010755603000000001 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.804047+0000", 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9590000000000001e-06 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.934993+0000", 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.13094678300000001 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.935051+0000", 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8128999999999999e-05 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.938270+0000", 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032187499999999998 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:150743 5.f 5:f8fc72c3:::benchmark_data_smithi064_147470_object18799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 391.704838317, 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27548363100000001, 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 150743 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812232+0000", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812231+0000", 2024-08-17T17:29:09.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812237+0000", 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.694e-06 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812248+0000", 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1012000000000001e-05 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.812253+0000", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2719999999999997e-06 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824221+0000", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011967396999999999 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.824354+0000", 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013296799999999999 2024-08-17T17:29:09.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988177+0000", 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.163823843 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988257+0000", 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9561000000000006e-05 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:37.988353+0000", 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5855999999999998e-05 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.004347+0000", 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015994228999999999 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.012117+0000", 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0077704050000000002 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083403+0000", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071285541999999993 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.083488+0000", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5028e-05 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:38.087716+0000", 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042280720000000003 2024-08-17T17:29:09.620 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:163575 5.f 5:fcf25f79:::benchmark_data_smithi064_147470_object31631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 368.99535477400002, 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25042483300000001, 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.621 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 163575 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521716+0000", 2024-08-17T17:29:09.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521715+0000", 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521720+0000", 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3839999999999996e-06 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521728+0000", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3000000000000002e-06 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.521733+0000", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0459999999999996e-06 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.536593+0000", 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014859387 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.536696+0000", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010328299999999999 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.649726+0000", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11302983899999999 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.649845+0000", 2024-08-17T17:29:09.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011899 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.650032+0000", 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000187159 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.657391+0000", 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0073592029999999999 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.674848+0000", 2024-08-17T17:29:09.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.017456639999999999 2024-08-17T17:29:09.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.761792+0000", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.086943764000000007 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.761865+0000", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3592999999999998e-05 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:00.772141+0000", 2024-08-17T17:29:09.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01027551 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:165359 5.f 5:faf23924:::benchmark_data_smithi064_147470_object33415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 366.113019958, 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27524345, 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.630 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 165359 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404051+0000", 2024-08-17T17:29:09.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404049+0000", 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404055+0000", 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9150000000000001e-06 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404069+0000", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4323e-05 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.404076+0000", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8040000000000004e-06 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.415728+0000", 2024-08-17T17:29:09.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011651729 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.415800+0000", 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1552999999999997e-05 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.606816+0000", 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.191016189 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.606935+0000", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000119521 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.607081+0000", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014616799999999999 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.627535+0000", 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.020453091999999999 2024-08-17T17:29:09.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.636155+0000", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0086204880000000008 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.675139+0000", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.038983775999999998 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.675200+0000", 2024-08-17T17:29:09.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1042000000000003e-05 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:03.679294+0000", 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040944279999999998 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:167570 5.f 5:fb418e04:::benchmark_data_smithi064_147470_object35626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:29:09.637 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 362.34916853599998, 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25951065200000001, 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167570 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:29:09.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167902+0000", 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167901+0000", 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167905+0000", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0060000000000003e-06 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167924+0000", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8298e-05 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.167931+0000", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8349999999999998e-06 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.182560+0000", 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014629647000000001 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.182671+0000", 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111246 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338341+0000", 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.15566972000000001 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338446+0000", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104911 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.338605+0000", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015865200000000001 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.346574+0000", 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079697080000000007 2024-08-17T17:29:09.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.346630+0000", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5788000000000002e-05 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.423542+0000", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.076911882000000001 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.423605+0000", 2024-08-17T17:29:09.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3003000000000006e-05 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:07.427413+0000", 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038078029999999998 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:176227 5.f 5:f2f7e856:::benchmark_data_smithi064_147470_object44283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 347.44766243700002, 2024-08-17T17:29:09.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29274401999999999, 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 176227 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069408+0000", 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069407+0000", 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069417+0000", 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7990000000000005e-06 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069427+0000", 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0095000000000001e-05 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.069434+0000", 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7340000000000002e-06 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.085581+0000", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.016147509000000001 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.085657+0000", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5525999999999995e-05 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.305768+0000", 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.22011108300000001 2024-08-17T17:29:09.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.305919+0000", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000151674 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.306089+0000", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016917399999999999 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.315695+0000", 2024-08-17T17:29:09.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096059730000000003 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.315736+0000", 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0970999999999998e-05 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.357779+0000", 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.042042989000000003 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.357818+0000", 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9650000000000002e-05 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:22.362152+0000", 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043341930000000001 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.651 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:196002 5.f 5:f3e23b7f:::benchmark_data_smithi064_147470_object64058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 315.467377521, 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24430998900000001, 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 196002 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.652 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049693+0000", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049692+0000", 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049703+0000", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1243000000000001e-05 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049715+0000", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2023e-05 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.049723+0000", 2024-08-17T17:29:09.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7090000000000002e-06 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.064549+0000", 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014826295999999999 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.064671+0000", 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121821 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236326+0000", 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.171655102 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236429+0000", 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000103484 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.236612+0000", 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018228100000000001 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.251683+0000", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015071319 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.254894+0000", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032105319999999999 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.290320+0000", 2024-08-17T17:29:09.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.035426080999999998 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.290383+0000", 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3212000000000004e-05 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:23:54.294003+0000", 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036203960000000001 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.658 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:209812 5.f 5:f9f68153:::benchmark_data_smithi064_147470_object77868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 293.50877695100002, 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.22315481500000001, 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:09.659 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 209812 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008294+0000", 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008293+0000", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008299+0000", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.942e-06 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008310+0000", 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1817999999999999e-05 2024-08-17T17:29:09.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.008317+0000", 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.029e-06 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.021703+0000", 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013385660000000001 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:09.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.021809+0000", 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010545800000000001 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203361+0000", 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.181552132 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203466+0000", 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105599 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.203617+0000", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015100700000000001 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.218447+0000", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014829956 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.218495+0000", 2024-08-17T17:29:09.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7503999999999998e-05 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.229017+0000", 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010522635000000001 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.229054+0000", 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6179e-05 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:24:16.231449+0000", 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002395112 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:149456 6.e 6:701be513:::benchmark_data_smithi064_185391_object41723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:28:29.706460+0000", 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 39.810610420000003, 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.65701571999999997, 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:09.666 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 149456 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706460+0000", 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:09.667 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706460+0000", 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706459+0000", 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706465+0000", 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0070000000000003e-06 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706476+0000", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0974e-05 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706481+0000", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.344e-06 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706535+0000", 2024-08-17T17:29:09.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3950999999999999e-05 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:29.706546+0000", 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0669999999999999e-05 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.334967+0000", 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.62842138599999997 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.335084+0000", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011641000000000001 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.335340+0000", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025652699999999999 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.337167+0000", 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0018269499999999999 2024-08-17T17:29:09.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.351142+0000", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01397494 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.363388+0000", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.012245859 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.363446+0000", 2024-08-17T17:29:09.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8149000000000002e-05 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:28:30.363476+0000", 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9830000000000001e-05 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:09.673 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:29:09.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:09.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:10.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:10.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:10.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:10.220 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:29:10.259 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:10.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:10.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:10.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:10.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:10.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:10.649 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:10.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:10 smithi175 ceph-mon[101434]: pgmap v660: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7289/76890 objects degraded (9.480%); 1505/76890 objects misplaced (1.957%) 2024-08-17T17:29:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3798185969' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[104516]: pgmap v660: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7289/76890 objects degraded (9.480%); 1505/76890 objects misplaced (1.957%) 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3798185969' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[109246]: pgmap v660: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 7289/76890 objects degraded (9.480%); 1505/76890 objects misplaced (1.957%) 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3798185969' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:10.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:10.774 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:10.818 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:29:10.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:10.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:11.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:11.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:11.049 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:11.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:11.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:11.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:11.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:11.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:11.422 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:11 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:11.422 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:11.422 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:11 smithi175 ceph-mon[101434]: osdmap e216: 8 total, 7 up, 7 in 2024-08-17T17:29:11.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:11.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:11.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:11.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[109246]: osdmap e216: 8 total, 7 up, 7 in 2024-08-17T17:29:11.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:11.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:11.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:11 smithi064 ceph-mon[104516]: osdmap e216: 8 total, 7 up, 7 in 2024-08-17T17:29:11.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:11.569 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:29:11.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:11.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:11.606 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:29:11.607 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:11.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:29:11.647 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:11.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:11.699 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:29:11.700 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:11.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:11.734 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:29:11.734 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:11.765 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:29:11.765 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:11.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:11.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:29:11.798 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:11.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:11.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:11.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:12.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:12.108 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:12.132 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:12.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:12.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:12.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:12.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:12.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:12 smithi064 ceph-mon[104516]: pgmap v662: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 6566/69201 objects degraded (9.488%); 1345/69201 objects misplaced (1.944%) 2024-08-17T17:29:12.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4014345948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:12.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:12 smithi064 ceph-mon[109246]: pgmap v662: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 6566/69201 objects degraded (9.488%); 1345/69201 objects misplaced (1.944%) 2024-08-17T17:29:12.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4014345948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:12.471 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:12 smithi175 ceph-mon[101434]: pgmap v662: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 6566/69201 objects degraded (9.488%); 1345/69201 objects misplaced (1.944%) 2024-08-17T17:29:12.471 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4014345948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:12.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:12.522 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:29:12.522 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:12.522 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-08-17T17:29:12.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd in 5 2024-08-17T17:29:12.546 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:12.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:12.727 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:12.728 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:12.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:12.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:12.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:12.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:29:13.076 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:29:13.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:13.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:13.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:13.301 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:13.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:13.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6566/69201 objects degraded (9.488%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6566/69201 objects degraded (9.488%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:13.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.532 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6566/69201 objects degraded (9.488%), 19 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:13.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:29:13.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:13.589 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:13.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:13.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:13.791 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:13.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:13.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:13.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:13.983 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:13] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:29:13.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:14.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:14.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:14.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:14.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:14.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:14.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:14.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:14.532 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:14.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:14.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:14 smithi175 ceph-mon[101434]: pgmap v663: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s; 6032/64470 objects degraded (9.356%); 1345/64470 objects misplaced (2.086%) 2024-08-17T17:29:14.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:14 smithi064 ceph-mon[109246]: pgmap v663: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s; 6032/64470 objects degraded (9.356%); 1345/64470 objects misplaced (2.086%) 2024-08-17T17:29:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:14 smithi064 ceph-mon[104516]: pgmap v663: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s; 6032/64470 objects degraded (9.356%); 1345/64470 objects misplaced (2.086%) 2024-08-17T17:29:14.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:14.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:14.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:14.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:15.064 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:15.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:15.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:15.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:15.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:15.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:29:15.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:15.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:15.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:15.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.754 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:15.812 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:15.812 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:15.829 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:15.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:15.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:15.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:16.142 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:29:16.020+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e216 definitely_dead 0 2024-08-17T17:29:16.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:16.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:16.299 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.5. 2024-08-17T17:29:16.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:16.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:16 smithi175 ceph-mon[101434]: pgmap v664: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4964/52272 objects degraded (9.496%); 982/52272 objects misplaced (1.879%) 2024-08-17T17:29:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4185179121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:16.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:29:16.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[104516]: pgmap v664: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4964/52272 objects degraded (9.496%); 982/52272 objects misplaced (1.879%) 2024-08-17T17:29:16.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4185179121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[109246]: pgmap v664: 109 pgs: 2 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 64 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 4964/52272 objects degraded (9.496%); 982/52272 objects misplaced (1.879%) 2024-08-17T17:29:16.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4185179121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T17:29:16.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:16.659 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.678 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:16.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:16.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:16.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:16.906 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:17.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:17.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:17.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:17.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:17.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:17.400 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T17:29:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:17 smithi175 ceph-mon[101434]: osdmap e217: 8 total, 7 up, 8 in 2024-08-17T17:29:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:17 smithi175 ceph-mon[101434]: osdmap e218: 8 total, 7 up, 8 in 2024-08-17T17:29:17.503 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:17.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.524 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:17.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:17.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:17.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:17.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:17.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T17:29:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[104516]: osdmap e217: 8 total, 7 up, 8 in 2024-08-17T17:29:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[104516]: osdmap e218: 8 total, 7 up, 8 in 2024-08-17T17:29:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T17:29:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[109246]: osdmap e217: 8 total, 7 up, 8 in 2024-08-17T17:29:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:17 smithi064 ceph-mon[109246]: osdmap e218: 8 total, 7 up, 8 in 2024-08-17T17:29:17.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.904 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:18.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:18.032 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:18.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.249 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:18.249 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:18.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:18.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:18.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.511 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:18.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:18.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:18 smithi175 ceph-mon[101434]: pgmap v666: 109 pgs: 1 peering, 2 active+clean+remapped, 23 active+undersized, 19 active+undersized+degraded, 64 active+clean; 998 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 4453/47754 objects degraded (9.325%); 982/47754 objects misplaced (2.056%) 2024-08-17T17:29:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4453/47754 objects degraded (9.325%), 19 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:18 smithi175 ceph-mon[101434]: osdmap e219: 8 total, 7 up, 8 in 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[104516]: pgmap v666: 109 pgs: 1 peering, 2 active+clean+remapped, 23 active+undersized, 19 active+undersized+degraded, 64 active+clean; 998 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 4453/47754 objects degraded (9.325%); 982/47754 objects misplaced (2.056%) 2024-08-17T17:29:18.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4453/47754 objects degraded (9.325%), 19 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:18.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[104516]: osdmap e219: 8 total, 7 up, 8 in 2024-08-17T17:29:18.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[109246]: pgmap v666: 109 pgs: 1 peering, 2 active+clean+remapped, 23 active+undersized, 19 active+undersized+degraded, 64 active+clean; 998 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 4453/47754 objects degraded (9.325%); 982/47754 objects misplaced (2.056%) 2024-08-17T17:29:18.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4453/47754 objects degraded (9.325%), 19 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:18.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:18 smithi064 ceph-mon[109246]: osdmap e219: 8 total, 7 up, 8 in 2024-08-17T17:29:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:18.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:18.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:19.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:29:19.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.369 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:19.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:19.419 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:19.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 216'10932 2024-08-17T17:29:19.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10920 2024-08-17T17:29:19.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.1a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10942 2024-08-17T17:29:19.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.1d starting backfill to osd.5 from (0'0,0'0] MAX to 216'11018 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8081 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8101 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7884 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.1c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11110 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11035 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[104516]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7889 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 216'10932 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10920 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.1a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10942 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.1d starting backfill to osd.5 from (0'0,0'0] MAX to 216'11018 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8081 2024-08-17T17:29:19.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8101 2024-08-17T17:29:19.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7884 2024-08-17T17:29:19.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.1c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11110 2024-08-17T17:29:19.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11035 2024-08-17T17:29:19.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:19 smithi064 ceph-mon[109246]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7889 2024-08-17T17:29:19.623 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-08-17T17:29:19.626 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 216'10932 2024-08-17T17:29:19.626 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10920 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.1a starting backfill to osd.5 from (0'0,0'0] MAX to 216'10942 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.1d starting backfill to osd.5 from (0'0,0'0] MAX to 216'11018 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8081 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 216'8101 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7884 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.1c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11110 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 216'11035 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:19 smithi175 ceph-mon[101434]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 216'7889 2024-08-17T17:29:19.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:19.692 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:19.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:19.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:19.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:19.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:20.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:20.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:20.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:20.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:20.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:20.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:20.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:20.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:20.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:20.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:20 smithi064 ceph-mon[104516]: pgmap v669: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 1 peering, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 22 active+undersized, 14 active+undersized+degraded, 63 active+clean; 934 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 4258/44703 objects degraded (9.525%); 2629/44703 objects misplaced (5.881%); 21 KiB/s, 4 objects/s recovering 2024-08-17T17:29:20.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:20 smithi064 ceph-mon[104516]: osdmap e220: 8 total, 7 up, 8 in 2024-08-17T17:29:20.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:20 smithi064 ceph-mon[109246]: pgmap v669: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 1 peering, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 22 active+undersized, 14 active+undersized+degraded, 63 active+clean; 934 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 4258/44703 objects degraded (9.525%); 2629/44703 objects misplaced (5.881%); 21 KiB/s, 4 objects/s recovering 2024-08-17T17:29:20.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:20 smithi064 ceph-mon[109246]: osdmap e220: 8 total, 7 up, 8 in 2024-08-17T17:29:20.632 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:20 smithi175 ceph-mon[101434]: pgmap v669: 109 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 1 peering, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 22 active+undersized, 14 active+undersized+degraded, 63 active+clean; 934 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 4258/44703 objects degraded (9.525%); 2629/44703 objects misplaced (5.881%); 21 KiB/s, 4 objects/s recovering 2024-08-17T17:29:20.632 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:20 smithi175 ceph-mon[101434]: osdmap e220: 8 total, 7 up, 8 in 2024-08-17T17:29:20.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.688 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:20.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:20.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:20.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:20.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:21.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:21.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:21.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:21.220 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:21.220 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:21.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:21.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:21.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:21.365 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:29:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:21.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:21.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:21.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:21.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:21 smithi175 ceph-mon[101434]: osdmap e221: 8 total, 7 up, 8 in 2024-08-17T17:29:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1072161728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[104516]: osdmap e221: 8 total, 7 up, 8 in 2024-08-17T17:29:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1072161728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[109246]: osdmap e221: 8 total, 7 up, 8 in 2024-08-17T17:29:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1072161728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:21.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:21.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:21.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:29:21.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:21.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:21.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:22.028 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:22.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:22.071 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:22.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:22.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:22.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:22.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:22.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:22.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:22.495 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:22 smithi175 ceph-mon[101434]: pgmap v672: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 863 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3893/41271 objects degraded (9.433%); 4597/41271 objects misplaced (11.139%); 64 KiB/s, 6 objects/s recovering 2024-08-17T17:29:22.495 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:22 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:22.495 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:22.495 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:22 smithi175 ceph-mon[101434]: osdmap e222: 8 total, 7 up, 8 in 2024-08-17T17:29:22.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:22.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[104516]: pgmap v672: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 863 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3893/41271 objects degraded (9.433%); 4597/41271 objects misplaced (11.139%); 64 KiB/s, 6 objects/s recovering 2024-08-17T17:29:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[104516]: osdmap e222: 8 total, 7 up, 8 in 2024-08-17T17:29:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[109246]: pgmap v672: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 8 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 863 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3893/41271 objects degraded (9.433%); 4597/41271 objects misplaced (11.139%); 64 KiB/s, 6 objects/s recovering 2024-08-17T17:29:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:22 smithi064 ceph-mon[109246]: osdmap e222: 8 total, 7 up, 8 in 2024-08-17T17:29:22.711 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:22.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:22.839 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:22.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:22.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:23.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:23.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:23.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:23.235 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:23.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:23.360 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:23.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:23.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:23.601 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3893/41271 objects degraded (9.433%), 15 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:23.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:23.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3893/41271 objects degraded (9.433%), 15 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3893/41271 objects degraded (9.433%), 15 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:23.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:23.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:23.862 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:23.862 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:23.878 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:23.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:24.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:24.042 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:24.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:24.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:24.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:24.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:24.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:24.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:24.376 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:23] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:29:24.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:24.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:24.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:24.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:24.546 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:24.624 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:29:24.624 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:24.624 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-17T17:29:24.624 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:29:24.624 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:29:24.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-17T17:29:24.647 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:24 smithi175 ceph-mon[101434]: pgmap v674: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 811 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 3614/38814 objects degraded (9.311%); 4598/38814 objects misplaced (11.846%); 55 KiB/s, 5 objects/s recovering 2024-08-17T17:29:24.647 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:24.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:24.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:24 smithi064 ceph-mon[104516]: pgmap v674: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 811 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 3614/38814 objects degraded (9.311%); 4598/38814 objects misplaced (11.846%); 55 KiB/s, 5 objects/s recovering 2024-08-17T17:29:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:24 smithi064 ceph-mon[109246]: pgmap v674: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 811 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 3614/38814 objects degraded (9.311%); 4598/38814 objects misplaced (11.846%); 55 KiB/s, 5 objects/s recovering 2024-08-17T17:29:24.700 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:29:24.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:24.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:24.840 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:24.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:24.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:25.045 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:25.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:25.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.823 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:25.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:25.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:25.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:26.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:26.083 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:26.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:26.117 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:29:26.118 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:26.157 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:29:26.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:26.208 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:29:26.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:26.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:26.241 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:29:26.241 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:26.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:26.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:29:26.277 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:26.309 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:29:26.310 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:26.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[104516]: pgmap v675: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 601 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 2727/28704 objects degraded (9.500%); 2814/28704 objects misplaced (9.804%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:29:26.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:26.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:26.310 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[109246]: pgmap v675: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 601 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 2727/28704 objects degraded (9.500%); 2814/28704 objects misplaced (9.804%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:29:26.311 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:26.311 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:26.311 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:26.342 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:26 smithi175 ceph-mon[101434]: pgmap v675: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 601 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 2727/28704 objects degraded (9.500%); 2814/28704 objects misplaced (9.804%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:29:26.342 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:26.342 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:26.343 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:29:26.343 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:26.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.394 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:26.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:26.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:26.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:26.593 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:29:26.593 INFO:teuthology.orchestra.run.smithi175.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:29:26.593 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:29:26.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:26.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:26.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:26.764 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:26.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:26.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:26.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/149265166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:27.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:27.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:27.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:27.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:27.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/149265166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:27.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/149265166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:27.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:27.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:27.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:29:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.596 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:27.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:27.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:27.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:27.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:27.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:27.952 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:27.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:27.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:28.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:28.125 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:28.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:28.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:28.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:28.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:28 smithi175 ceph-mon[101434]: pgmap v676: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 519 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 2305/24783 objects degraded (9.301%); 2814/24783 objects misplaced (11.355%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:29:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:28 smithi064 ceph-mon[104516]: pgmap v676: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 519 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 2305/24783 objects degraded (9.301%); 2814/24783 objects misplaced (11.355%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:29:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:28 smithi064 ceph-mon[109246]: pgmap v676: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 519 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 2305/24783 objects degraded (9.301%); 2814/24783 objects misplaced (11.355%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:29:28.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:28.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:28.636 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:28.664 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:28.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:28.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:28.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:28.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:28.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:29.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:131905 5.9 5:97548a8e:::benchmark_data_smithi064_147470_object131904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:02.019824+0000", 2024-08-17T17:29:29.069 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.04565700799998, 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3528637200000002, 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 131905 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019824+0000", 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.070 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019824+0000", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019824+0000", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019878+0000", 2024-08-17T17:29:29.071 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3786999999999998e-05 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019885+0000", 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8159999999999996e-06 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.019889+0000", 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6659999999999999e-06 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.033499+0000", 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013609662 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.033633+0000", 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000133829 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.033778+0000", 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014552799999999999 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.043423+0000", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0096451109999999996 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.049209+0000", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0057854899999999999 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.060808+0000", 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011598858 2024-08-17T17:29:29.074 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.060847+0000", 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9651999999999999e-05 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049368+0000", 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98852083199999996 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.075 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049380+0000", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2150999999999999e-05 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.372608+0000", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3232278800000001 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.372688+0000", 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9856000000000003e-05 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.076 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:131908 5.9 5:9cbfadaf:::benchmark_data_smithi064_147470_object131907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:02.022449+0000", 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.04303189299998, 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3541361570000001, 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 131908 2024-08-17T17:29:29.077 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022449+0000", 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022449+0000", 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.078 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022448+0000", 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022503+0000", 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4540000000000003e-05 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022516+0000", 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.379e-05 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.022521+0000", 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1019999999999996e-06 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.038508+0000", 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015986897999999999 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.080 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.038844+0000", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00033552199999999999 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.038993+0000", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014863699999999999 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.043768+0000", 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0047753240000000001 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.081 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.060878+0000", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.017110468 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.060888+0000", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1849999999999993e-06 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049434+0000", 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98854672799999999 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.082 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.083 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.083 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049444+0000", 2024-08-17T17:29:29.083 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8030000000000008e-06 2024-08-17T17:29:29.083 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.083 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.376511+0000", 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.327067403 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.084 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.376586+0000", 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4181e-05 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:131929 5.9 5:90deab0f:::benchmark_data_smithi064_147470_object131928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:02.065122+0000", 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.00035910499997, 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.338912434, 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.085 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 131929 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065122+0000", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.086 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065122+0000", 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065121+0000", 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065151+0000", 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9736000000000001e-05 2024-08-17T17:29:29.087 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065160+0000", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.859e-06 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065168+0000", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.204e-06 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065201+0000", 2024-08-17T17:29:29.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3237e-05 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.065210+0000", 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0690000000000008e-06 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049469+0000", 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98425901500000001 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.089 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049476+0000", 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3320000000000003e-06 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.376639+0000", 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.327163198 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.376782+0000", 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014285400000000001 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.090 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.376989+0000", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00020625600000000001 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.384303+0000", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0073146260000000003 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.384315+0000", 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1588e-05 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.091 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.400314+0000", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015999454999999999 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.400355+0000", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0559999999999998e-05 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.404035+0000", 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036798999999999998 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.092 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:131932 5.9 5:95f95bbb:::benchmark_data_smithi064_147470_object131931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:02.067957+0000", 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 446.99752457, 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.339934188, 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:29.093 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 131932 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.067957+0000", 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.067957+0000", 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.094 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.067956+0000", 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.067999+0000", 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3183000000000003e-05 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.095 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.068016+0000", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6239000000000002e-05 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.068023+0000", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4259999999999997e-06 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.068068+0000", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5272999999999998e-05 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.096 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.068078+0000", 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0074999999999999e-05 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049509+0000", 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98143082999999998 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049517+0000", 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.661e-06 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.097 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.380806+0000", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.331288842 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.380934+0000", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012847600000000001 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.381079+0000", 2024-08-17T17:29:29.098 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014501299999999999 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.387455+0000", 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0063762920000000004 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.387504+0000", 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8405999999999997e-05 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.099 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.404097+0000", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.016593411999999998 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.404167+0000", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9956999999999994e-05 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.407891+0000", 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037237899999999998 2024-08-17T17:29:29.100 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.34626.0:131938 5.9 5:9f1020cf:::benchmark_data_smithi064_147470_object131937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:22:02.075628+0000", 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 446.98985323800002, 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3375718490000001, 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.34626", 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:29:29.101 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 131938 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075628+0000", 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075628+0000", 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.102 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075627+0000", 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075694+0000", 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6761000000000007e-05 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075704+0000", 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0417e-05 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075711+0000", 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5660000000000003e-06 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075760+0000", 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8575000000000002e-05 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.104 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:02.075770+0000", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0078000000000001e-05 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049538+0000", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.97376850599999998 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:03.049545+0000", 2024-08-17T17:29:29.105 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.17e-06 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.384340+0000", 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3347947360000001 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.384481+0000", 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014056800000000001 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.106 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.384669+0000", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018861 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.387529+0000", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00285966 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.387595+0000", 2024-08-17T17:29:29.107 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6299999999999999e-05 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.409231+0000", 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.021635969000000001 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.409298+0000", 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7148000000000002e-05 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.108 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:22:04.413200+0000", 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003901631 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72143 6.15 6:a9ad64ed:::benchmark_data_smithi064_185391_object72142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.72342995899999, 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.940724318, 2024-08-17T17:29:29.109 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72143 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.110 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342050+0000", 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.111 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342077+0000", 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7270000000000001e-05 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342092+0000", 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4783999999999999e-05 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342099+0000", 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8870000000000003e-06 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.112 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342136+0000", 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7243000000000002e-05 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342143+0000", 2024-08-17T17:29:29.113 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6950000000000003e-06 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.153747+0000", 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81160374499999999 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.153759+0000", 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1907e-05 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.114 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278060+0000", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.124301014 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278182+0000", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012255999999999999 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278289+0000", 2024-08-17T17:29:29.115 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000106573 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281551+0000", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032626669999999999 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281583+0000", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1698e-05 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281591+0000", 2024-08-17T17:29:29.116 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9920000000000007e-06 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281628+0000", 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6358e-05 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282776+0000", 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011481849999999999 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.117 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72144 6.15 6:ab663db7:::benchmark_data_smithi064_185391_object72143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.71866694600001, 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.941616505, 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72144 2024-08-17T17:29:29.118 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.119 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346813+0000", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346869+0000", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5569000000000003e-05 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346882+0000", 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3366e-05 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.120 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346888+0000", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6810000000000003e-06 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346927+0000", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8804000000000003e-05 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346935+0000", 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0099999999999995e-06 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.121 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154103+0000", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.80716819699999998 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154109+0000", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0939999999999996e-06 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280421+0000", 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.126312218 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.122 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280512+0000", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0737999999999998e-05 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280608+0000", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6014999999999995e-05 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282931+0000", 2024-08-17T17:29:29.123 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023226420000000002 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282940+0000", 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6860000000000001e-06 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.287152+0000", 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042121260000000001 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.124 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.287190+0000", 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7676999999999999e-05 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288431+0000", 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001240903 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72163 6.15 6:afaee3c3:::benchmark_data_smithi064_185391_object72162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:29:29.125 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.66077098100001, 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.892213878, 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72163 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:29:29.126 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.127 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.127 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.127 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:29.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404709+0000", 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.129 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404752+0000", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3374000000000002e-05 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404764+0000", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2225000000000001e-05 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404771+0000", 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6460000000000004e-06 2024-08-17T17:29:29.130 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404813+0000", 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2273e-05 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404822+0000", 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2819999999999996e-06 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.131 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154171+0000", 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.74934978299999999 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154180+0000", 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3820000000000004e-06 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282979+0000", 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1287988870000001 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.132 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.283134+0000", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000155616 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.283301+0000", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016718599999999999 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288488+0000", 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051863259999999998 2024-08-17T17:29:29.133 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288494+0000", 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3860000000000004e-06 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295291+0000", 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0067970110000000004 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.134 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295355+0000", 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4337000000000004e-05 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.296924+0000", 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0015688290000000001 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72199 6.15 6:ac10834f:::benchmark_data_smithi064_185391_object72198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:29:29.135 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.61626489, 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8490388900000001, 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72199 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.136 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449215+0000", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449260+0000", 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4409000000000001e-05 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.137 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449270+0000", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0675e-05 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449277+0000", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449335+0000", 2024-08-17T17:29:29.138 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7503000000000002e-05 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449346+0000", 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0767999999999999e-05 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154204+0000", 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70485822300000001 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.139 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154209+0000", 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8110000000000002e-06 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284450+0000", 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.13024152 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284593+0000", 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142776 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.140 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284753+0000", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016029999999999999 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288500+0000", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003746494 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288532+0000", 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2416999999999998e-05 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.141 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.297006+0000", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0084740590000000008 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.297060+0000", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3687000000000003e-05 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298255+0000", 2024-08-17T17:29:29.142 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011954139999999999 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72213 6.15 6:aa0bc0e7:::benchmark_data_smithi064_185391_object72212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.60312640500001, 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8406289010000001, 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.143 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72213 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.144 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462354+0000", 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462421+0000", 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7621999999999997e-05 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.145 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462434+0000", 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2686e-05 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462439+0000", 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9690000000000002e-06 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462479+0000", 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9821999999999999e-05 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.146 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462486+0000", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0090000000000001e-06 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154233+0000", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.69174695500000005 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154241+0000", 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9990000000000008e-06 2024-08-17T17:29:29.147 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.285915+0000", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1316743970000001 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.286033+0000", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000117416 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.286175+0000", 2024-08-17T17:29:29.148 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142476 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288562+0000", 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023867889999999998 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288567+0000", 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8879999999999996e-06 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.149 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.301655+0000", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013088416 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.301712+0000", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6478999999999999e-05 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.302984+0000", 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001272118 2024-08-17T17:29:29.150 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72242 6.15 6:aef0d0cc:::benchmark_data_smithi064_185391_object72241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.57913491599999, 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8174085250000001, 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72242 2024-08-17T17:29:29.151 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:29:29.152 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486458+0000", 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112156 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486465+0000", 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.939e-06 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.153 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486469+0000", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2309999999999998e-06 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486501+0000", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1683999999999997e-05 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486508+0000", 2024-08-17T17:29:29.154 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6470000000000001e-06 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154266+0000", 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.66775841199999997 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154270+0000", 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2599999999999999e-06 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.155 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288587+0000", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1343165909999999 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288695+0000", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010791 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288799+0000", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104202 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.156 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292495+0000", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036959290000000001 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292533+0000", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8092000000000001e-05 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303032+0000", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01049899 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.157 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303069+0000", 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6548000000000003e-05 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303755+0000", 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068645100000000003 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72252 6.15 6:ac5ac986:::benchmark_data_smithi064_185391_object72251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.158 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.569726855, 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.808657811, 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72252 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.159 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495753+0000", 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.160 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495804+0000", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0812000000000001e-05 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495816+0000", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1687e-05 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495842+0000", 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6234e-05 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.161 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495878+0000", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6211999999999998e-05 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495885+0000", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4799999999999998e-06 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154296+0000", 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.65841167099999998 2024-08-17T17:29:29.162 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154303+0000", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.289677+0000", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1353735899999999 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.289861+0000", 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018380100000000001 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.163 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.290033+0000", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017293000000000001 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292541+0000", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025071519999999999 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292574+0000", 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3439000000000003e-05 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.164 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303834+0000", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011259748 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303866+0000", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2712999999999997e-05 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304412+0000", 2024-08-17T17:29:29.165 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054580899999999996 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72299 6.15 6:a8c7d24b:::benchmark_data_smithi064_185391_object72298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.52207750400001, 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.761700287, 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.166 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72299 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.167 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543403+0000", 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543462+0000", 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8947000000000003e-05 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.168 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543472+0000", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8120000000000002e-06 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543476+0000", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7079999999999996e-06 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543515+0000", 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8852e-05 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.169 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543523+0000", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8779999999999998e-06 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154355+0000", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.61083240000000005 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154363+0000", 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2389999999999996e-06 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.170 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291104+0000", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1367416850000001 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291259+0000", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000154337 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291420+0000", 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000160851 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.171 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295162+0000", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037425610000000002 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295222+0000", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0024e-05 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304526+0000", 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093043810000000005 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.172 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304577+0000", 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0447e-05 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305104+0000", 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00052719400000000003 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72344 6.15 6:aacad454:::benchmark_data_smithi064_185391_object72343:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.173 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.47764248999999, 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.721979817, 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72344 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:29:29.174 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587838+0000", 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.175 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587951+0000", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112562 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587960+0000", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587966+0000", 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9059999999999999e-06 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.176 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.588017+0000", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1319000000000001e-05 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.588027+0000", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6989999999999998e-06 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154387+0000", 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56636031200000003 2024-08-17T17:29:29.177 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154394+0000", 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292667+0000", 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.138272902 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.178 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292811+0000", 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000143845 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292995+0000", 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018369400000000001 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295212+0000", 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022173050000000001 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.179 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295229+0000", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7161999999999999e-05 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.308328+0000", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013098426 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.308377+0000", 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9567999999999997e-05 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.180 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309819+0000", 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014416019999999999 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72396 6.15 6:ab62b94c:::benchmark_data_smithi064_185391_object72395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.36117981699999, 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.609453421, 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.181 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72396 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.182 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704301+0000", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704360+0000", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9194999999999998e-05 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704373+0000", 2024-08-17T17:29:29.183 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2911999999999999e-05 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704380+0000", 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1980000000000004e-06 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704418+0000", 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8041999999999999e-05 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.184 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704424+0000", 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.268e-06 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154429+0000", 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45000499599999999 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154438+0000", 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0010000000000007e-06 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.185 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294019+0000", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.139580679 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294145+0000", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012642600000000001 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294242+0000", 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6430999999999999e-05 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.186 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298309+0000", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040671079999999998 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298317+0000", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1550000000000007e-06 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.312477+0000", 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014159916 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.187 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.312535+0000", 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7658000000000003e-05 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313755+0000", 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001220418 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.188 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72430 6.15 6:a946c8e2:::benchmark_data_smithi064_185391_object72429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.324323892, 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5734773990000002, 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72430 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.189 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741156+0000", 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.190 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741208+0000", 2024-08-17T17:29:29.191 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1790999999999999e-05 2024-08-17T17:29:29.191 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741219+0000", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1229999999999999e-05 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741224+0000", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1599999999999997e-06 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741267+0000", 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2917000000000003e-05 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741274+0000", 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6120000000000004e-06 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154483+0000", 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.41320958299999999 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154493+0000", 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0259e-05 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298342+0000", 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1438485890000001 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298501+0000", 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015924900000000001 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.196 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298681+0000", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000180046 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303798+0000", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051164660000000001 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304429+0000", 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00063155899999999996 2024-08-17T17:29:29.197 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313814+0000", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093843699999999995 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313870+0000", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6700000000000003e-05 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.314635+0000", 2024-08-17T17:29:29.198 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00076452599999999996 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.199 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72541 6.15 6:a961463b:::benchmark_data_smithi064_185391_object72540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:29 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2305/24783 objects degraded (9.301%), 14 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:29.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:29 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2305/24783 objects degraded (9.301%), 14 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.190808928, 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4433222699999999, 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72541 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.201 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874671+0000", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874754+0000", 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2688999999999994e-05 2024-08-17T17:29:29.202 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874767+0000", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3389e-05 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874772+0000", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1989999999999997e-06 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874831+0000", 2024-08-17T17:29:29.203 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8782000000000001e-05 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874843+0000", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1446e-05 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154528+0000", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27968510499999999 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154539+0000", 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1076e-05 2024-08-17T17:29:29.204 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300346+0000", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1458074700000001 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300524+0000", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017809 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300662+0000", 2024-08-17T17:29:29.205 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000137816 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304437+0000", 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003775011 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304481+0000", 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3884000000000002e-05 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.206 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.316283+0000", 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011801936000000001 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.316359+0000", 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6062000000000005e-05 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.317995+0000", 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001635584 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.207 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72617 6.15 6:ab07db17:::benchmark_data_smithi064_185391_object72616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 145.01243470899999, 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2663090640000001, 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72617 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.208 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053046+0000", 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.209 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053146+0000", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010018600000000001 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053157+0000", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0451e-05 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053162+0000", 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2109999999999997e-06 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.210 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053200+0000", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8426000000000001e-05 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053218+0000", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7501999999999999e-05 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154575+0000", 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.101357234 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.211 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154584+0000", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8710000000000003e-06 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305134+0000", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1505501330000001 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305234+0000", 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010040899999999999 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.212 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305334+0000", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9568999999999995e-05 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309844+0000", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0045095530000000003 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309891+0000", 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7169000000000002e-05 2024-08-17T17:29:29.213 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.318049+0000", 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0081580650000000008 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.318091+0000", 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2663000000000001e-05 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.214 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319356+0000", 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0012642319999999999 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72663 6.15 6:ae8c21f0:::benchmark_data_smithi064_185391_object72662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 144.942782962, 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.198120447, 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.215 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72663 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.216 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122696+0000", 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122724+0000", 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7628999999999999e-05 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.217 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122737+0000", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.322e-05 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122744+0000", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1910000000000003e-06 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122804+0000", 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9978000000000001e-05 2024-08-17T17:29:29.218 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122817+0000", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.255e-05 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154621+0000", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.031804333999999997 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154632+0000", 2024-08-17T17:29:29.219 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0482999999999999e-05 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306247+0000", 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.151615823 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306371+0000", 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012342899999999999 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.220 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306491+0000", 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012012 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309930+0000", 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0034388779999999998 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309938+0000", 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1159999999999999e-06 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.221 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319434+0000", 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0094958760000000003 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319484+0000", 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0108999999999997e-05 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.222 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320819+0000", 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001334911 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72666 6.15 6:afe642e1:::benchmark_data_smithi064_185391_object72665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 144.93767980499999, 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.194315598, 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:29:29.223 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72666 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:29:29.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127793+0000", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999919 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127838+0000", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4923999999999999e-05 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127853+0000", 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4499e-05 2024-08-17T17:29:29.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127859+0000", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7619999999999998e-06 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127899+0000", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9755000000000001e-05 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127906+0000", 2024-08-17T17:29:29.226 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9800000000000001e-06 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154668+0000", 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.026761565000000001 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154676+0000", 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0090000000000007e-06 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307367+0000", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.152691165 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307506+0000", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013873800000000001 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307675+0000", 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016890500000000001 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309944+0000", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022697709999999999 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309996+0000", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1415000000000002e-05 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320880+0000", 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010883840000000001 2024-08-17T17:29:29.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320931+0000", 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1153999999999998e-05 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.322117+0000", 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011863659999999999 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.230 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:29:29.231 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:29:29.231 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:29:29.231 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:29.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:29.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:29.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:29.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:29 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2305/24783 objects degraded (9.301%), 14 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:29.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:29.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:29.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.668 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:29.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:29.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:29.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:29.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:30.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:30.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:29:30.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:30.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:30 smithi064 ceph-mon[104516]: pgmap v677: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 429 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 2026/20451 objects degraded (9.907%); 2155/20451 objects misplaced (10.537%); 2.6 MiB/s, 73 objects/s recovering 2024-08-17T17:29:30.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:30 smithi064 ceph-mon[109246]: pgmap v677: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 429 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 2026/20451 objects degraded (9.907%); 2155/20451 objects misplaced (10.537%); 2.6 MiB/s, 73 objects/s recovering 2024-08-17T17:29:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:30.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:30.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:30.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:30 smithi175 ceph-mon[101434]: pgmap v677: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 429 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 2026/20451 objects degraded (9.907%); 2155/20451 objects misplaced (10.537%); 2.6 MiB/s, 73 objects/s recovering 2024-08-17T17:29:30.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:30.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:30.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:30.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:30.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:30.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:30.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:30.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:30.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:30.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:31.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:31.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:31.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:31.356 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:31.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:31.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.536 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:29:31.537 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:29:31.608 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:31.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.659 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:31.725 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:31.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:31.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:31.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:31.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.195 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:32.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:32.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:32 smithi064 ceph-mon[104516]: pgmap v678: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 338 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 1545/16068 objects degraded (9.615%); 1433/16068 objects misplaced (8.918%); 2.2 MiB/s, 63 objects/s recovering 2024-08-17T17:29:32.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:32 smithi064 ceph-mon[109246]: pgmap v678: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 338 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 1545/16068 objects degraded (9.615%); 1433/16068 objects misplaced (8.918%); 2.2 MiB/s, 63 objects/s recovering 2024-08-17T17:29:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:32.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:32 smithi175 ceph-mon[101434]: pgmap v678: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 338 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 1545/16068 objects degraded (9.615%); 1433/16068 objects misplaced (8.918%); 2.2 MiB/s, 63 objects/s recovering 2024-08-17T17:29:32.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:32.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:32.524 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:29:32.524 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:32.524 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-17T17:29:32.524 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:29:32.524 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:29:32.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-17T17:29:32.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:32.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:32.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:32.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:32.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:32.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:32.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:33.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:33.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:33.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:33.255 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.280 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:33.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:33.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.429 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:33.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:33.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:33.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:33.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:33.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:33.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:33.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:33.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:33.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:33.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:33.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:33.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:34.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 107730 objects 2024-08-17T17:29:34.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :109.965 2024-08-17T17:29:34.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:34.114 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:34.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:34.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:34 smithi064 ceph-mon[104516]: pgmap v679: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 267 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1164/12684 objects degraded (9.177%); 1433/12684 objects misplaced (11.298%); 1.9 MiB/s, 54 objects/s recovering 2024-08-17T17:29:34.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:34 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1164/12684 objects degraded (9.177%), 14 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:34.250 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:33] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.51.0" 2024-08-17T17:29:34.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:34 smithi064 ceph-mon[109246]: pgmap v679: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 267 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1164/12684 objects degraded (9.177%); 1433/12684 objects misplaced (11.298%); 1.9 MiB/s, 54 objects/s recovering 2024-08-17T17:29:34.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:34 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1164/12684 objects degraded (9.177%), 14 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:34.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:34.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:34 smithi175 ceph-mon[101434]: pgmap v679: 109 pgs: 5 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 16 active+undersized, 8 active+undersized+degraded, 64 active+clean; 267 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1164/12684 objects degraded (9.177%); 1433/12684 objects misplaced (11.298%); 1.9 MiB/s, 54 objects/s recovering 2024-08-17T17:29:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:34 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1164/12684 objects degraded (9.177%), 14 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:34.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.488 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:34.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:34.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.612 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:34.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:34.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:34.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:34.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:34.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:34.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:34.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:35.087 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:35.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:35.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:35.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:35.300 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-08-17T17:29:35.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-08-17T17:29:35.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:35.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:35.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:35.490 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:35.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:35.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:35.616 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:35.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:35.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:35.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:35.862 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:35.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.025 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:36.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:36.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:36.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:36.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:36.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:36 smithi175 ceph-mon[101434]: pgmap v680: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 17 active+undersized, 7 active+undersized+degraded, 64 active+clean; 64 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 400/2937 objects degraded (13.619%); 60/2937 objects misplaced (2.043%); 1.9 MiB/s, 83 objects/s recovering 2024-08-17T17:29:36.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:36 smithi064 ceph-mon[104516]: pgmap v680: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 17 active+undersized, 7 active+undersized+degraded, 64 active+clean; 64 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 400/2937 objects degraded (13.619%); 60/2937 objects misplaced (2.043%); 1.9 MiB/s, 83 objects/s recovering 2024-08-17T17:29:36.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:36 smithi064 ceph-mon[109246]: pgmap v680: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 17 active+undersized, 7 active+undersized+degraded, 64 active+clean; 64 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 400/2937 objects degraded (13.619%); 60/2937 objects misplaced (2.043%); 1.9 MiB/s, 83 objects/s recovering 2024-08-17T17:29:36.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:36.519 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:36.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:36.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:36.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:36.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:36.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:36.893 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:36.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:37.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:37.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:37.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:37.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:37.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:37.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:37.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:37.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:37.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:37.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:37.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:37.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:37.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:37.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:37.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:37.892 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:37.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:38.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:38.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:38.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:38.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:38.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:38.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:38.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:38 smithi175 ceph-mon[101434]: pgmap v681: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 17 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 143/678 objects degraded (21.091%); 60/678 objects misplaced (8.850%); 528 KiB/s, 62 objects/s recovering 2024-08-17T17:29:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:38 smithi064 ceph-mon[104516]: pgmap v681: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 17 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 143/678 objects degraded (21.091%); 60/678 objects misplaced (8.850%); 528 KiB/s, 62 objects/s recovering 2024-08-17T17:29:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:38 smithi064 ceph-mon[109246]: pgmap v681: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 17 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 143/678 objects degraded (21.091%); 60/678 objects misplaced (8.850%); 528 KiB/s, 62 objects/s recovering 2024-08-17T17:29:38.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:38.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:38.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:38.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:38.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:38.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:38.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:38.883 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:29:38.884 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:29:38.884 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2024-08-17T17:29:38.884 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:29:38.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:38.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:38.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:39.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:39.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.137 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:39.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:39 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 143/678 objects degraded (21.091%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:39.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:39 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 143/678 objects degraded (21.091%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:39.238 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:39.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:39.324 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:39.324 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:39.362 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:39.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:39 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 143/678 objects degraded (21.091%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:39.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:39.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:39.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:39.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:39.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:39.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:39.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:39.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:39.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:39.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:39.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:39.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:40.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:40.003 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:40.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:29:40.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:40.125 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:29:40.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:40.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:40.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:40.264 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[104516]: pgmap v682: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 528 KiB/s, 171 objects/s recovering 2024-08-17T17:29:40.264 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:40.264 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:40.264 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[109246]: pgmap v682: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 528 KiB/s, 171 objects/s recovering 2024-08-17T17:29:40.264 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:40.265 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:40.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:40.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:40 smithi175 ceph-mon[101434]: pgmap v682: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 528 KiB/s, 171 objects/s recovering 2024-08-17T17:29:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:40.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:40.506 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:40.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:40.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:40.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:40.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:40.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:40.877 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:40.979 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:41.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:41.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:41.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:41.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:41.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:41.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:41.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:41 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:41 smithi175 ceph-mon[101434]: osdmap e223: 8 total, 7 up, 8 in 2024-08-17T17:29:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[104516]: osdmap e223: 8 total, 7 up, 8 in 2024-08-17T17:29:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3934053479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:41 smithi064 ceph-mon[109246]: osdmap e223: 8 total, 7 up, 8 in 2024-08-17T17:29:41.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:41.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:41.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:41.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:41.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:41.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:41.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:41.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:41.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:41.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:42.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:42.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:42.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:42.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:42.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:42.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:42 smithi175 ceph-mon[101434]: pgmap v684: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 168 objects/s recovering 2024-08-17T17:29:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:42 smithi175 ceph-mon[101434]: osdmap e224: 8 total, 7 up, 8 in 2024-08-17T17:29:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:42 smithi064 ceph-mon[104516]: pgmap v684: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 168 objects/s recovering 2024-08-17T17:29:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:42 smithi064 ceph-mon[104516]: osdmap e224: 8 total, 7 up, 8 in 2024-08-17T17:29:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:42 smithi064 ceph-mon[109246]: pgmap v684: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 168 objects/s recovering 2024-08-17T17:29:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:42 smithi064 ceph-mon[109246]: osdmap e224: 8 total, 7 up, 8 in 2024-08-17T17:29:42.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:42.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:42.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:42.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:42.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:42.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:42.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:42.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:42.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:42.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:42.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.012 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:43 smithi064 ceph-mon[104516]: osdmap e225: 8 total, 7 up, 8 in 2024-08-17T17:29:43.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:43 smithi064 ceph-mon[109246]: osdmap e225: 8 total, 7 up, 8 in 2024-08-17T17:29:43.240 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:43.264 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:43.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:43.367 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:43.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:43 smithi175 ceph-mon[101434]: osdmap e225: 8 total, 7 up, 8 in 2024-08-17T17:29:43.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:43.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:43.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:43.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:43.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:43.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:43.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:43.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:43.894 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:43.958 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:43] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:29:43.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:44.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:29:44.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.019 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:44.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:44.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:44.139 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:29:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:29:44.139 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:29:44.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:44.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:44.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:44.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[104516]: pgmap v687: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 218 objects/s recovering 2024-08-17T17:29:44.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[104516]: 6.c scrub starts 2024-08-17T17:29:44.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[104516]: 6.c scrub ok 2024-08-17T17:29:44.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 138/450 objects degraded (30.667%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:44.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[109246]: pgmap v687: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 218 objects/s recovering 2024-08-17T17:29:44.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[109246]: 6.c scrub starts 2024-08-17T17:29:44.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[109246]: 6.c scrub ok 2024-08-17T17:29:44.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:44 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 138/450 objects degraded (30.667%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:44.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:44.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.396 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:44.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:44 smithi175 ceph-mon[101434]: pgmap v687: 109 pgs: 2 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 64 active+clean; 12 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 138/450 objects degraded (30.667%); 44/450 objects misplaced (9.778%); 0 B/s, 218 objects/s recovering 2024-08-17T17:29:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:44 smithi175 ceph-mon[101434]: 6.c scrub starts 2024-08-17T17:29:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:44 smithi175 ceph-mon[101434]: 6.c scrub ok 2024-08-17T17:29:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:44 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 138/450 objects degraded (30.667%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:44.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:44.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:44.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:44.647 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:44.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:44.774 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:44.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:44.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:44.900 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:45.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:45.153 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:45.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.283 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:45.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:45 smithi175 ceph-mon[101434]: 6.1c scrub starts 2024-08-17T17:29:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:45 smithi175 ceph-mon[101434]: 6.1c scrub ok 2024-08-17T17:29:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:45 smithi064 ceph-mon[104516]: 6.1c scrub starts 2024-08-17T17:29:45.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:45 smithi064 ceph-mon[104516]: 6.1c scrub ok 2024-08-17T17:29:45.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:45 smithi064 ceph-mon[109246]: 6.1c scrub starts 2024-08-17T17:29:45.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:45 smithi064 ceph-mon[109246]: 6.1c scrub ok 2024-08-17T17:29:45.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.540 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:45.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.670 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.799 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:45.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:45.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:46.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:46.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:46.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:46.177 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[104516]: 6.a scrub starts 2024-08-17T17:29:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[104516]: 6.a scrub ok 2024-08-17T17:29:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[104516]: pgmap v688: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 4 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 67 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 99/363 objects degraded (27.273%); 15/363 objects misplaced (4.132%); 0 B/s, 174 objects/s recovering 2024-08-17T17:29:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[109246]: 6.a scrub starts 2024-08-17T17:29:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[109246]: 6.a scrub ok 2024-08-17T17:29:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:46 smithi064 ceph-mon[109246]: pgmap v688: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 4 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 67 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 99/363 objects degraded (27.273%); 15/363 objects misplaced (4.132%); 0 B/s, 174 objects/s recovering 2024-08-17T17:29:46.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:46.304 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:46.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:46 smithi175 ceph-mon[101434]: 6.a scrub starts 2024-08-17T17:29:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:46 smithi175 ceph-mon[101434]: 6.a scrub ok 2024-08-17T17:29:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:46 smithi175 ceph-mon[101434]: pgmap v688: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 4 active+remapped+backfill_wait, 19 active+undersized, 5 active+undersized+degraded, 67 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 99/363 objects degraded (27.273%); 15/363 objects misplaced (4.132%); 0 B/s, 174 objects/s recovering 2024-08-17T17:29:46.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:46.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:46.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:46.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:46.762 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:46.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:46.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:46.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:46.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:47.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:47.062 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:47.162 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' removed 2024-08-17T17:29:47.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:47.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:47.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:47 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:29:47.266 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:47 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:29:47.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.312 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:47 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:29:47.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:47.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:47.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:47.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:47.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:47.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:47.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:47.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:47.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:48.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:48.191 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:48.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:48.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:48.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[104516]: pgmap v689: 109 pgs: 14 active+clean+remapped, 19 active+undersized, 5 active+undersized+degraded, 71 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 91/363 objects degraded (25.069%); 15/363 objects misplaced (4.132%); 0 B/s, 152 objects/s recovering 2024-08-17T17:29:48.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:29:48.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[104516]: osdmap e226: 8 total, 7 up, 8 in 2024-08-17T17:29:48.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[109246]: pgmap v689: 109 pgs: 14 active+clean+remapped, 19 active+undersized, 5 active+undersized+degraded, 71 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 91/363 objects degraded (25.069%); 15/363 objects misplaced (4.132%); 0 B/s, 152 objects/s recovering 2024-08-17T17:29:48.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:29:48.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:48 smithi064 ceph-mon[109246]: osdmap e226: 8 total, 7 up, 8 in 2024-08-17T17:29:48.398 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:48.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:48.429 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:29:48.430 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:29:48.431 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-08-17T17:29:48.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-08-17T17:29:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:48.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:48.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:48 smithi175 ceph-mon[101434]: pgmap v689: 109 pgs: 14 active+clean+remapped, 19 active+undersized, 5 active+undersized+degraded, 71 active+clean; 10 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 91/363 objects degraded (25.069%); 15/363 objects misplaced (4.132%); 0 B/s, 152 objects/s recovering 2024-08-17T17:29:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1096887852' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:29:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:48 smithi175 ceph-mon[101434]: osdmap e226: 8 total, 7 up, 8 in 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:48.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:48.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:29:48.681 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:48.691 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:48.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:29:48.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:48.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:29:48.715 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:48.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:48.732 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:29:48.732 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:48.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:48.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:48.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:49.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:49.061 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:49.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:49.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:49.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:49.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:49.308 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:49.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:49.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:49 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 91/363 objects degraded (25.069%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2333645368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:49 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 91/363 objects degraded (25.069%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2333645368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:49.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:49.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:49 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 91/363 objects degraded (25.069%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2333645368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:49.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:49.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:49.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:49.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:49.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:50.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:50.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:50.156 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:50.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:50.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:50.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:50.408 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:50.431 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:50.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:50 smithi064 ceph-mon[104516]: pgmap v691: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:50.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:50 smithi064 ceph-mon[109246]: pgmap v691: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:29:50.529 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:29:50.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:50.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:50 smithi175 ceph-mon[101434]: pgmap v691: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:50.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:50.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:50.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:50.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:50.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:50.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:50.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.966 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:29:50.966 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:50.966 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:29:50.966 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2024-08-17T17:29:50.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T17:29:50.978 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:50.978 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:51.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:51.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:51.157 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:51.182 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:51.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:51.305 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:51.407 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:51.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:51.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:51.557 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:51.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:51.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:51.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:51.806 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:51.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:51.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:52.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:52.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:52.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:52.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:52.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:52 smithi064 ceph-mon[104516]: pgmap v692: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:52.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:52 smithi064 ceph-mon[109246]: pgmap v692: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:52.281 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:52.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:52.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:52.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:52.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:52.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:52.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:52 smithi175 ceph-mon[101434]: pgmap v692: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:52.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.688 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:52.790 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.815 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:52.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:29:52.853 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:52.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:53.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:53.180 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:29:53.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:53.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.319 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:53.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1251489253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:53 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1251489253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:53 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:53.544 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:53.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1251489253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:53 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:53.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:53.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:53.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:53.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:53.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:53.949 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:54.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:54.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:54.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:54.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:54.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:54.249 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:29:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:29:53] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:29:54.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[104516]: pgmap v693: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:54.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:54.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[104516]: osdmap e227: 8 total, 7 up, 8 in 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[109246]: pgmap v693: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[109246]: osdmap e227: 8 total, 7 up, 8 in 2024-08-17T17:29:54.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:54 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:54.303 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:54.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:54.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:54.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:54.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:54.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:54 smithi175 ceph-mon[101434]: pgmap v693: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:54 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:54 smithi175 ceph-mon[101434]: osdmap e227: 8 total, 7 up, 8 in 2024-08-17T17:29:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:54 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:29:54.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:54.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:54.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:54.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:54.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:54.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:55.044 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:55.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:55.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:55.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:55.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:55.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:55.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:55.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:55.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:55.561 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:55.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:56.066 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:56.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:56.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[104516]: pgmap v695: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:56.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:56.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:56.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2518659513' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[109246]: pgmap v695: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:56.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:56.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:56.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2518659513' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:56 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.181 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' created 2024-08-17T17:29:56.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:56.215 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:56.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:56.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:56.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:56 smithi175 ceph-mon[101434]: pgmap v695: 73 pgs: 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:56 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2518659513' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:56 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:29:56.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:56.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:56.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:56.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:56.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:56.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.956 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:56.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:57.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:57.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:57.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:57.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:57.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:57.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:57.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:57.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:57 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:29:57.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:57 smithi064 ceph-mon[104516]: osdmap e228: 8 total, 7 up, 8 in 2024-08-17T17:29:57.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:57 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:29:57.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:57 smithi064 ceph-mon[109246]: osdmap e228: 8 total, 7 up, 8 in 2024-08-17T17:29:57.457 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:57.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:29:57.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:57.580 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:57.629 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:57 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:29:57.629 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:57 smithi175 ceph-mon[101434]: osdmap e228: 8 total, 7 up, 8 in 2024-08-17T17:29:57.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:57.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:57.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:57.829 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:57.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:57.923 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:57.930 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:57.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:58.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:58.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:58.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:58.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:58.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:58.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[109246]: pgmap v697: 89 pgs: 1 creating+peering, 15 unknown, 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:58.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:58.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[109246]: osdmap e229: 8 total, 7 up, 8 in 2024-08-17T17:29:58.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[104516]: pgmap v697: 89 pgs: 1 creating+peering, 15 unknown, 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:58.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:58.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:58 smithi064 ceph-mon[104516]: osdmap e229: 8 total, 7 up, 8 in 2024-08-17T17:29:58.320 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:58.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-08-17T17:29:58.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:58.442 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:58.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:58.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:58.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:58 smithi175 ceph-mon[101434]: pgmap v697: 89 pgs: 1 creating+peering, 15 unknown, 9 active+clean+remapped, 16 active+undersized, 2 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:29:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:58 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:58 smithi175 ceph-mon[101434]: osdmap e229: 8 total, 7 up, 8 in 2024-08-17T17:29:58.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:58.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:58.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:58.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:58.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:58.961 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:59.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:59.073 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:29:59.073 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:59.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:59.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:59.097 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:29:59.098 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:59.116 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:29:59.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:59.132 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:29:59.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:59.149 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:29:59.149 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:59.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:59.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:59.184 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:29:59.184 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:59.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.222 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:59.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:59.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: osdmap e230: 8 total, 7 up, 8 in 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4251060211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/836183771' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: osdmap e230: 8 total, 7 up, 8 in 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:59.316 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.317 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.317 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4251060211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:59.317 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/836183771' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.317 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:29:59 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:59.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:59.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:59.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:59.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:59.593 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: osdmap e230: 8 total, 7 up, 8 in 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4251060211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/836183771' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:29:59 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:29:59.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:59.705 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T17:29:59.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:59.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:59.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:29:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:59.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:59.967 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:29:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:00.002 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:00.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:00.092 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:00.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:00.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:00.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:00.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:00.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pgmap v700: 89 pgs: 1 creating+peering, 8 unknown, 9 active+clean+remapped, 17 active+undersized, 2 active+undersized+degraded, 52 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:30:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: osdmap e231: 8 total, 7 up, 8 in 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized; 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: osd.6 (root=default,host=smithi175) is down 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 1.0 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 2.6 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 2.7 is stuck undersized for 2m, current state active+undersized, last acting [7,2] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.0 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.3 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.a is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.11 is stuck undersized for 2m, current state active+undersized, last acting [7,4] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.14 is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,7] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-08-17T17:30:00.412 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.5 is stuck undersized for 2m, current state active+undersized, last acting [4,7] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.a is stuck undersized for 2m, current state active+undersized, last acting [2,7] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,3] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.e is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [3,1] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [3,7] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.13 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: pg 4.18 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: application not enabled on pool 'unique_pool_2' 2024-08-17T17:30:00.413 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:00 smithi175 ceph-mon[101434]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pgmap v700: 89 pgs: 1 creating+peering, 8 unknown, 9 active+clean+remapped, 17 active+undersized, 2 active+undersized+degraded, 52 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: osdmap e231: 8 total, 7 up, 8 in 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized; 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: osd.6 (root=default,host=smithi175) is down 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 1.0 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 2.6 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 2.7 is stuck undersized for 2m, current state active+undersized, last acting [7,2] 2024-08-17T17:30:00.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.0 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.3 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.a is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.11 is stuck undersized for 2m, current state active+undersized, last acting [7,4] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.14 is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,7] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.5 is stuck undersized for 2m, current state active+undersized, last acting [4,7] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.a is stuck undersized for 2m, current state active+undersized, last acting [2,7] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,3] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.e is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [3,1] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [3,7] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.13 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: pg 4.18 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: application not enabled on pool 'unique_pool_2' 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[104516]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pgmap v700: 89 pgs: 1 creating+peering, 8 unknown, 9 active+clean+remapped, 17 active+undersized, 2 active+undersized+degraded, 52 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: osdmap e231: 8 total, 7 up, 8 in 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized; 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: osd.6 (root=default,host=smithi175) is down 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 2 pgs degraded, 18 pgs undersized 2024-08-17T17:30:00.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 1.0 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 2.6 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 2.7 is stuck undersized for 2m, current state active+undersized, last acting [7,2] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.0 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.3 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.a is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.11 is stuck undersized for 2m, current state active+undersized, last acting [7,4] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.14 is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,7] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [3,2] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.5 is stuck undersized for 2m, current state active+undersized, last acting [4,7] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.a is stuck undersized for 2m, current state active+undersized, last acting [2,7] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,3] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.e is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [3,1] 2024-08-17T17:30:00.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [3,7] 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.12 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.13 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: pg 4.18 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: application not enabled on pool 'unique_pool_2' 2024-08-17T17:30:00.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:00 smithi064 ceph-mon[109246]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:30:00.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:00.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:00.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:00.590 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:00.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:00.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:00.768 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:00.768 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:00.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:00.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:01.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:01.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:01.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:01.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:01.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:01.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:01.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:01.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:01.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:01.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:01.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:01.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:01.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:01.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:01.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:01.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:01.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:02.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:02.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:02.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:02.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:02.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:02.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:02.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.329 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:02 smithi175 ceph-mon[101434]: pgmap v702: 89 pgs: 1 creating+peering, 1 unknown, 9 active+clean+remapped, 20 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:02.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:02.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:02.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:02 smithi064 ceph-mon[104516]: pgmap v702: 89 pgs: 1 creating+peering, 1 unknown, 9 active+clean+remapped, 20 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:02.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:02 smithi064 ceph-mon[109246]: pgmap v702: 89 pgs: 1 creating+peering, 1 unknown, 9 active+clean+remapped, 20 active+undersized, 2 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:02.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:02.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:02.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:02.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:02.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:02.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.944 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:03.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:03.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:03.169 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:03.190 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:03.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:03.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:03.313 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:03.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140775 5.5 5:abc89a9a:::benchmark_data_smithi064_147470_object8831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:30:03.434 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.25844133599998, 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1096915650000001, 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140775 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172352+0000", 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172354+0000", 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.843e-06 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172361+0000", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1069999999999999e-06 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172374+0000", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3142999999999999e-05 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172382+0000", 2024-08-17T17:30:03.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1859999999999992e-06 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188729+0000", 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016346823 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188871+0000", 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141722 2024-08-17T17:30:03.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239525+0000", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050654413000000002 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239649+0000", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012354 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.239798+0000", 2024-08-17T17:30:03.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014950799999999999 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.252143+0000", 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012344554000000001 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.252150+0000", 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9240000000000001e-06 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.272388+0000", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.020237721 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.272596+0000", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020802900000000001 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281984+0000", 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093880230000000005 2024-08-17T17:30:03.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.282044+0000", 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0028999999999997e-05 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140784 5.5 5:af152c95:::benchmark_data_smithi064_147470_object8840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:30:03.442 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.24287036800001, 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.093724857, 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140784 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:30:03.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187923+0000", 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187925+0000", 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0899999999999999e-06 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187929+0000", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0339999999999998e-06 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187940+0000", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.082e-05 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.187946+0000", 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3300000000000004e-06 2024-08-17T17:30:03.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.188969+0000", 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010226149999999999 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.189359+0000", 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038968800000000001 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.189775+0000", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041620399999999999 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.250918+0000", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.061142672000000002 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.250931+0000", 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3404000000000001e-05 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271603+0000", 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0206717270000001 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271704+0000", 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010087200000000001 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281568+0000", 2024-08-17T17:30:03.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098639290000000004 2024-08-17T17:30:03.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.281648+0000", 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0471999999999998e-05 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176891 5.3 5:c4b25f19:::benchmark_data_smithi064_147470_object44947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 400.31852829799999, 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.021502071, 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.450 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176891 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112265+0000", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112267+0000", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.283e-06 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112272+0000", 2024-08-17T17:30:03.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4660000000000001e-06 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112283+0000", 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.083e-05 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112289+0000", 2024-08-17T17:30:03.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.015e-06 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112488+0000", 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019912400000000001 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.112514+0000", 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5990999999999999e-05 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126288+0000", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013773979 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126405+0000", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117107 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.126520+0000", 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114629 2024-08-17T17:30:03.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133526+0000", 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070063629999999998 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133602+0000", 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6087999999999994e-05 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133673+0000", 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0895999999999993e-05 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133724+0000", 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0569999999999999e-05 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133767+0000", 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3730000000000003e-05 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.457 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176912 5.3 5:c2336cb6:::benchmark_data_smithi064_147470_object44968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 400.27698781100003, 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98830326099999999, 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:30:03.458 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176912 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153806+0000", 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5699999999999998e-07 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153808+0000", 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3300000000000001e-06 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153815+0000", 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4289999999999998e-06 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.153820+0000", 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.203e-06 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.159421+0000", 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056008500000000001 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.159432+0000", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1501000000000001e-05 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130238+0000", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97080522899999999 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130361+0000", 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012380099999999999 2024-08-17T17:30:03.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.130454+0000", 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2539000000000003e-05 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133661+0000", 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032068109999999999 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.137820+0000", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041596580000000001 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142004+0000", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041835850000000001 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142062+0000", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7779999999999999e-05 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142109+0000", 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7088e-05 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:176939 5.3 5:c2db8817:::benchmark_data_smithi064_147470_object44995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 400.236419915, 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95685923799999995, 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 176939 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194374+0000", 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194375+0000", 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.24e-06 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194380+0000", 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:30:03.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194399+0000", 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9885999999999999e-05 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194407+0000", 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3050000000000004e-06 2024-08-17T17:30:03.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194613+0000", 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000206255 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:23.194625+0000", 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1481000000000001e-05 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.133903+0000", 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93927822900000002 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.134036+0000", 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133688 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.134155+0000", 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011839 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142119+0000", 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079641769999999994 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.142165+0000", 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5624000000000002e-05 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151114+0000", 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089488929999999994 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151184+0000", 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0588999999999998e-05 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:23:24.151233+0000", 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8721000000000003e-05 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.472 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76618 6.10 6:0b484596:::benchmark_data_smithi064_185391_object76617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.245982398, 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99620523500000002, 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76618 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184812+0000", 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.389e-06 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184858+0000", 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5833e-05 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184868+0000", 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7380000000000006e-06 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184875+0000", 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9099999999999999e-06 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184980+0000", 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104818 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184990+0000", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0491e-05 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176539+0000", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99154894900000001 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176667+0000", 2024-08-17T17:30:03.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012793799999999999 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176758+0000", 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0721000000000001e-05 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180859+0000", 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041008249999999998 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180907+0000", 2024-08-17T17:30:03.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8519000000000001e-05 2024-08-17T17:30:03.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180943+0000", 2024-08-17T17:30:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.480 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:03.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:03.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5848e-05 2024-08-17T17:30:03.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180976+0000", 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3101e-05 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181016+0000", 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0154999999999998e-05 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76627 6.10 6:089e1d75:::benchmark_data_smithi064_185391_object76626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.235578804, 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98607286000000005, 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76627 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.483 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195217+0000", 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2309999999999999e-06 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195263+0000", 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6406999999999998e-05 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195291+0000", 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8021000000000002e-05 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195302+0000", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1068e-05 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195400+0000", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7590000000000006e-05 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195423+0000", 2024-08-17T17:30:03.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2714e-05 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178205+0000", 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98278234499999995 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178295+0000", 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0185000000000005e-05 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178394+0000", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8790999999999997e-05 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181104+0000", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002710329 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181147+0000", 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2651999999999998e-05 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181195+0000", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7511e-05 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181248+0000", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3599e-05 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181288+0000", 2024-08-17T17:30:03.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9416999999999999e-05 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76681 6.10 6:0d6e3b1e:::benchmark_data_smithi064_185391_object76680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.246800+0000", 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.18399350799999, 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.934700537, 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.490 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76681 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246800+0000", 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246800+0000", 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246801+0000", 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.358e-06 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246827+0000", 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6067e-05 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246839+0000", 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1413000000000001e-05 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.246845+0000", 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3670000000000002e-06 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.247645+0000", 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079941500000000004 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.247654+0000", 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.179593+0000", 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93193881300000003 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.179710+0000", 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117068 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.179839+0000", 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012885300000000001 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181345+0000", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001506322 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181358+0000", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.296e-05 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181406+0000", 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7944e-05 2024-08-17T17:30:03.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181466+0000", 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0186999999999998e-05 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181501+0000", 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4545999999999998e-05 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76708 6.10 6:0e16ef80:::benchmark_data_smithi064_185391_object76707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.163517442, 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98798299199999995, 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.498 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76708 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267277+0000", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267298+0000", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0469999999999999e-05 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267307+0000", 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8509999999999994e-06 2024-08-17T17:30:03.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267314+0000", 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2759999999999997e-06 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267375+0000", 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1606000000000006e-05 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267386+0000", 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.092e-05 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181547+0000", 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91416100199999994 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181662+0000", 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011430900000000001 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181785+0000", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012363599999999999 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.184737+0000", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029515589999999999 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.187694+0000", 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029570989999999999 2024-08-17T17:30:03.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255151+0000", 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.067457153000000006 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255214+0000", 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3188999999999994e-05 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255259+0000", 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4844e-05 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76736 6.10 6:0da5d701:::benchmark_data_smithi064_185391_object76735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.287925+0000", 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.14286835199999, 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96749383700000002, 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.505 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76736 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287925+0000", 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287925+0000", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287927+0000", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7489999999999999e-06 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287950+0000", 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2742e-05 2024-08-17T17:30:03.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287968+0000", 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8522999999999999e-05 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.287975+0000", 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1149999999999996e-06 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.288080+0000", 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104786 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.288091+0000", 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0842999999999999e-05 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.183087+0000", 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89499572699999996 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.183200+0000", 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113748 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.183354+0000", 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015382699999999999 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.187699+0000", 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043451469999999997 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.187741+0000", 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1501999999999997e-05 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255340+0000", 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.067599324000000002 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255383+0000", 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3197e-05 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255419+0000", 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5607e-05 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101324 6.a 6:595ddac9:::benchmark_data_smithi064_185391_object101323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 148.23520317800001, 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99253369000000002, 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101324 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.513 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195592+0000", 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.776e-06 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195616+0000", 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3910999999999999e-05 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195629+0000", 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2731000000000001e-05 2024-08-17T17:30:03.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195637+0000", 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9589999999999995e-06 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195694+0000", 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7405999999999999e-05 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195703+0000", 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9129999999999992e-06 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.167031+0000", 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97132766800000003 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169828+0000", 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027970429999999999 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169985+0000", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015776000000000001 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183860+0000", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013874600000000001 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187988+0000", 2024-08-17T17:30:03.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041284069999999997 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.188043+0000", 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4528999999999999e-05 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.188124+0000", 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0987000000000002e-05 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101328 6.a 6:517c2a0b:::benchmark_data_smithi064_185391_object101327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 148.23320296099999, 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001870466, 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101328 2024-08-17T17:30:03.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197592+0000", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3540000000000001e-06 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197617+0000", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4729e-05 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197635+0000", 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.806e-05 2024-08-17T17:30:03.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197643+0000", 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4740000000000005e-06 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197711+0000", 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7844999999999998e-05 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197720+0000", 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9279999999999999e-06 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174342+0000", 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97662202799999998 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174493+0000", 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150719 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174642+0000", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014933700000000001 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183902+0000", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092596199999999997 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199315+0000", 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015413524 2024-08-17T17:30:03.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199372+0000", 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6468000000000002e-05 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199461+0000", 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9380000000000001e-05 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101343 6.10 6:0d818121:::benchmark_data_smithi064_185391_object101342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 148.22154813500001, 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.007587639, 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101343 2024-08-17T17:30:03.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209247+0000", 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.544e-06 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209270+0000", 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2912e-05 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209288+0000", 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8263000000000001e-05 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209297+0000", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8289999999999997e-06 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209365+0000", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8204e-05 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209375+0000", 2024-08-17T17:30:03.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8810000000000008e-06 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.153546+0000", 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94417129 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.153557+0000", 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0428000000000001e-05 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211140+0000", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057583081 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211300+0000", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015991200000000001 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211416+0000", 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116729 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215779+0000", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043626170000000001 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215800+0000", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0744e-05 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215946+0000", 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146479 2024-08-17T17:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216011+0000", 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4697999999999996e-05 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216833+0000", 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082202800000000004 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.534 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101358 6.a 6:572067d6:::benchmark_data_smithi064_185391_object101357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 148.203792706, 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97542105499999998, 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101358 2024-08-17T17:30:03.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227003+0000", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7969999999999999e-06 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227030+0000", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7667000000000001e-05 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227048+0000", 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8164000000000001e-05 2024-08-17T17:30:03.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227058+0000", 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3070000000000008e-06 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227136+0000", 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8450999999999998e-05 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227149+0000", 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2662e-05 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178318+0000", 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95116961899999997 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178451+0000", 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013217900000000001 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178606+0000", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155582 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.184332+0000", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057253010000000003 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202269+0000", 2024-08-17T17:30:03.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017937278000000001 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202329+0000", 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0217000000000003e-05 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202422+0000", 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2831000000000006e-05 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101373 6.10 6:09660f81:::benchmark_data_smithi064_185391_object101372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 147.235678412, 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0313173980000001, 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.542 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101373 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195117+0000", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4279999999999999e-06 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195150+0000", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3856000000000002e-05 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195162+0000", 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1605e-05 2024-08-17T17:30:03.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195170+0000", 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8140000000000001e-06 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195231+0000", 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1317999999999998e-05 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195239+0000", 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3420000000000008e-06 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.213792+0000", 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018552329 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.213943+0000", 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015166299999999999 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.214108+0000", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164346 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.219035+0000", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049269159999999999 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.219079+0000", 2024-08-17T17:30:03.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4691999999999998e-05 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225354+0000", 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006274572 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225415+0000", 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0992999999999997e-05 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.226432+0000", 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010175239999999999 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101396 6.10 6:09c77aec:::benchmark_data_smithi064_185391_object101395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 147.21700311500001, 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0184236339999999, 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.549 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101396 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213791+0000", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000001e-07 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214115+0000", 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032435899999999998 2024-08-17T17:30:03.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214126+0000", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0180000000000001e-05 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214133+0000", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1799999999999999e-06 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.215566+0000", 2024-08-17T17:30:03.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001433024 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.215575+0000", 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0880000000000001e-06 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216893+0000", 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001317784 2024-08-17T17:30:03.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.217076+0000", 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000182936 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.217259+0000", 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018311099999999999 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225311+0000", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080522219999999995 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.226530+0000", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012188430000000001 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.231197+0000", 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046669459999999999 2024-08-17T17:30:03.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.231246+0000", 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9564999999999997e-05 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.232214+0000", 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00096778000000000001 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149426 6.a 6:54a57211:::benchmark_data_smithi064_185391_object41693:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 94.108110189000001, 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.060886048, 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149426 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.557 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322684+0000", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8800000000000001e-07 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322688+0000", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1330000000000001e-06 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322700+0000", 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2087e-05 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322708+0000", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8169999999999999e-06 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533973+0000", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21126436700000001 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533980+0000", 2024-08-17T17:30:03.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4789999999999999e-06 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.383499+0000", 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.849518569 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.383569+0000", 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0708000000000001e-05 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149429 6.a 6:573fa82c:::benchmark_data_smithi064_185391_object41696:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 94.108073028000007, 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.062352024, 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149429 2024-08-17T17:30:03.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322721+0000", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.06e-07 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322724+0000", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5440000000000001e-06 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322730+0000", 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1460000000000001e-06 2024-08-17T17:30:03.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322736+0000", 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1920000000000002e-06 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533997+0000", 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.211261265 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.534002+0000", 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8339999999999998e-06 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.384980+0000", 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85097835600000005 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.385072+0000", 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2181000000000004e-05 2024-08-17T17:30:03.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149442 6.1a 6:597706b7:::benchmark_data_smithi064_185391_object41709:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 94.108051492000001, 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.081072644, 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.567 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149442 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7100000000000002e-07 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322745+0000", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2129999999999998e-06 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322751+0000", 2024-08-17T17:30:03.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1800000000000001e-06 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322756+0000", 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8590000000000004e-06 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535270+0000", 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21251468400000001 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535279+0000", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2109999999999998e-06 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386083+0000", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.850804059 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386255+0000", 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017275799999999999 2024-08-17T17:30:03.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386548+0000", 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029291700000000001 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.393934+0000", 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073854960000000001 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403675+0000", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097411210000000002 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403730+0000", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4724e-05 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403815+0000", 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4950999999999994e-05 2024-08-17T17:30:03.574 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149446 6.1a 6:585de6ab:::benchmark_data_smithi064_185391_object41713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 94.094609841999997, 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0677979660000001, 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:30:03.575 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149446 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:30:03.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336186+0000", 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.153e-06 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336192+0000", 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1510000000000001e-06 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:30:03.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336216+0000", 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4536e-05 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336228+0000", 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1873e-05 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535297+0000", 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19906828200000001 2024-08-17T17:30:03.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535302+0000", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8640000000000004e-06 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388247+0000", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85294507799999997 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388349+0000", 2024-08-17T17:30:03.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102496 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388578+0000", 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0002287 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.395844+0000", 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072662600000000001 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403850+0000", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080058560000000004 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403907+0000", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7241999999999998e-05 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403982+0000", 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4474999999999994e-05 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.581 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.582 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.582 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:30:03.582 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:30:03.582 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:30:03.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:03.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:03.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:03.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:03.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:03.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:03.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:03.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:03.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:04.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:04.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:04.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:04.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:04.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:04.235 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:03] "GET /metrics HTTP/1.1" 200 34285 "" "Prometheus/2.51.0" 2024-08-17T17:30:04.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:04.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:04.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:04.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:04.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:04.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:04.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:04.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:04.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:04.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.859 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:04 smithi175 ceph-mon[101434]: pgmap v703: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:04 smithi064 ceph-mon[104516]: pgmap v703: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:04 smithi064 ceph-mon[109246]: pgmap v703: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:04.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.984 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:05.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:05.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:05.211 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:05.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:05.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:05.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:05.449 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:05.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:05.587 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:05.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:05.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:05.839 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:05.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:05.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:05.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:06.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:30:06.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:06.118 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:06.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:06.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:06.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:06.327 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:06.327 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:06.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:06.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:06.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:06.497 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:06.599 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:06.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:06.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:06.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:06.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:06.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:06.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:06 smithi064 ceph-mon[104516]: pgmap v704: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:06.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:06 smithi064 ceph-mon[109246]: pgmap v704: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:06 smithi175 ceph-mon[101434]: pgmap v704: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:06.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:07.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:07.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:07.256 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:07.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:07.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.509 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.635 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:30:07.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:07.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:07.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.888 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:07.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:07.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:07.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:08.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:08.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:08.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:08.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:08.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:08.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:08.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:08.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:08.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:08.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:08.516 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:08.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:08.638 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-08-17T17:30:08.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:08.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:08.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:08.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:08.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:08.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:08 smithi175 ceph-mon[101434]: pgmap v705: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:08 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:08 smithi175 ceph-mon[101434]: osdmap e232: 8 total, 7 up, 8 in 2024-08-17T17:30:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:08.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[104516]: pgmap v705: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[104516]: osdmap e232: 8 total, 7 up, 8 in 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[109246]: pgmap v705: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1843188822' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:08.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[109246]: osdmap e232: 8 total, 7 up, 8 in 2024-08-17T17:30:08.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:08.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.146 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.273 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:09.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:09.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:09.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:09.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:09 smithi175 ceph-mon[101434]: osdmap e233: 8 total, 7 up, 8 in 2024-08-17T17:30:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:09 smithi064 ceph-mon[104516]: osdmap e233: 8 total, 7 up, 8 in 2024-08-17T17:30:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1455791087' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:09 smithi064 ceph-mon[109246]: osdmap e233: 8 total, 7 up, 8 in 2024-08-17T17:30:10.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:10.026 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:10.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.212 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:30:10.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:10.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:30:10.229 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:10.245 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:30:10.245 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:10.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:10.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:30:10.264 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:10.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:10.281 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:30:10.281 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:10.298 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:30:10.298 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:10.299 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:10.315 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:30:10.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:10.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:10.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:10.523 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:10.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:10.647 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:10.702 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-08-17T17:30:10.703 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:30:10.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:30:10.705 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:30:10.705 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:30:10.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:10.772 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:30:10.772 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:30:10.772 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_210574 2024-08-17T17:30:10.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:30:10.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:30:10.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:10.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:10 smithi175 ceph-mon[101434]: pgmap v708: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:10 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1790230421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[104516]: pgmap v708: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1790230421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[109246]: pgmap v708: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6/84 objects degraded (7.143%); 7/84 objects misplaced (8.333%) 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1790230421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:10.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:30:11.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:11.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:11.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:11.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:11.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:11.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:11.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:11.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:11.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:11.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:11.655 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:11.692 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:11.692 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:11.712 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:11.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:11.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2932 2916 182.2 182.25 0.00385212 0.00545792 2024-08-17T17:30:11.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:11.877 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:11.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:11.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:11 smithi064 ceph-mon[104516]: pgmap v709: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 5.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 104 KiB/s wr, 1 op/s; 6/123 objects degraded (4.878%); 7/123 objects misplaced (5.691%) 2024-08-17T17:30:11.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:11 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6/123 objects degraded (4.878%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:11.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:11 smithi064 ceph-mon[109246]: pgmap v709: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 5.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 104 KiB/s wr, 1 op/s; 6/123 objects degraded (4.878%); 7/123 objects misplaced (5.691%) 2024-08-17T17:30:11.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:11 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6/123 objects degraded (4.878%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:11.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:12.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:12.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:11 smithi175 ceph-mon[101434]: pgmap v709: 89 pgs: 9 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 57 active+clean; 5.4 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 104 KiB/s wr, 1 op/s; 6/123 objects degraded (4.878%); 7/123 objects misplaced (5.691%) 2024-08-17T17:30:12.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:11 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6/123 objects degraded (4.878%), 2 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:12.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:12.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:12.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:12.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.420 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:12.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:12.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:12.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:12.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:12.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:12.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:12.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6017 6001 187.497 192.812 0.00323608 0.00532028 2024-08-17T17:30:12.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:12.984 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:13.086 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:13.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:13.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:13.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:13.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.407 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:13.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:13.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:13.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:13.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8688 8672 180.637 166.938 0.00758907 0.00552673 2024-08-17T17:30:13.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:13.899 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:13.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:14.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:14.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:14.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:14.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:14.289 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:13] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:30:14.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:14 smithi064 ceph-mon[109246]: pgmap v710: 89 pgs: 9 active+clean+remapped, 20 active+undersized, 3 active+undersized+degraded, 57 active+clean; 18 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.7 MiB/s wr, 26 op/s; 156/729 objects degraded (21.399%); 7/729 objects misplaced (0.960%) 2024-08-17T17:30:14.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:14 smithi064 ceph-mon[104516]: pgmap v710: 89 pgs: 9 active+clean+remapped, 20 active+undersized, 3 active+undersized+degraded, 57 active+clean; 18 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.7 MiB/s wr, 26 op/s; 156/729 objects degraded (21.399%); 7/729 objects misplaced (0.960%) 2024-08-17T17:30:14.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:14.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:14 smithi175 ceph-mon[101434]: pgmap v710: 89 pgs: 9 active+clean+remapped, 20 active+undersized, 3 active+undersized+degraded, 57 active+clean; 18 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 1.7 MiB/s wr, 26 op/s; 156/729 objects degraded (21.399%); 7/729 objects misplaced (0.960%) 2024-08-17T17:30:14.435 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:14.461 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:14.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:14.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:14.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:14.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:14.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 11333 11317 176.789 165.312 0.00835222 0.00564938 2024-08-17T17:30:14.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:14.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:14.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:15.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:15.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:15.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:15.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:15.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:15.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:15.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:15.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13567 13551 169.352 139.625 0.00274945 0.00585868 2024-08-17T17:30:15.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:15.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:15.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.116 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:16.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:16.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:16.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:16.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:16 smithi175 ceph-mon[101434]: pgmap v711: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 482 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 2700/23001 objects degraded (11.739%); 7/23001 objects misplaced (0.030%) 2024-08-17T17:30:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:16 smithi064 ceph-mon[104516]: pgmap v711: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 482 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 2700/23001 objects degraded (11.739%); 7/23001 objects misplaced (0.030%) 2024-08-17T17:30:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:16 smithi064 ceph-mon[109246]: pgmap v711: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 482 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 2700/23001 objects degraded (11.739%); 7/23001 objects misplaced (0.030%) 2024-08-17T17:30:16.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:16.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:16.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 15701 15685 163.355 133.375 0.00142629 0.00608023 2024-08-17T17:30:16.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:16.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:16.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:17.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:17.028 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:17.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:17.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:17.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:17.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:17.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:17.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:17 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3507/30351 objects degraded (11.555%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:17.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:17.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:17.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:17 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3507/30351 objects degraded (11.555%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:17.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:17 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3507/30351 objects degraded (11.555%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:17.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:17.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:17.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:17.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:17.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 17801 17785 158.767 131.25 0.00709646 0.00629199 2024-08-17T17:30:17.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:17.827 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:17.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:17.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:18.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:18.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:18.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:18.191 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:18.212 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:18.314 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:18.337 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:18 smithi175 ceph-mon[101434]: pgmap v712: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 635 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3507/30351 objects degraded (11.555%); 7/30351 objects misplaced (0.023%) 2024-08-17T17:30:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:18 smithi064 ceph-mon[104516]: pgmap v712: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 635 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3507/30351 objects degraded (11.555%); 7/30351 objects misplaced (0.023%) 2024-08-17T17:30:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:18 smithi064 ceph-mon[109246]: pgmap v712: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 635 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 3507/30351 objects degraded (11.555%); 7/30351 objects misplaced (0.023%) 2024-08-17T17:30:18.439 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:18.465 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:18.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:18.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:18.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:18.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:18.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:18.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 19568 19552 152.725 110.438 0.00465238 0.006542 2024-08-17T17:30:18.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:18.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:19.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:19.117 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:19.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:19.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:19.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:19.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:19.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:19.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:19.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:19.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.664 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:19.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:19.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:19.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 21542 21526 149.462 123.375 0.00620265 0.00662454 2024-08-17T17:30:19.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:19.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:19.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:19.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:20.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:20.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:20.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:20.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:20.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:20.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:20.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:20.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:20 smithi175 ceph-mon[101434]: pgmap v713: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 939 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 4204/44931 objects degraded (9.357%); 7/44931 objects misplaced (0.016%) 2024-08-17T17:30:20.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:20.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:20 smithi064 ceph-mon[104516]: pgmap v713: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 939 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 4204/44931 objects degraded (9.357%); 7/44931 objects misplaced (0.016%) 2024-08-17T17:30:20.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:20 smithi064 ceph-mon[109246]: pgmap v713: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 939 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 4204/44931 objects degraded (9.357%); 7/44931 objects misplaced (0.016%) 2024-08-17T17:30:20.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:20.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:20.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:20.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:20.775 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 22941 22925 143.259 87.4375 0.012659 0.00697396 2024-08-17T17:30:20.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:20.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:20.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:20.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:21.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.068 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:21.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:21.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:21.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:21.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:21.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:21.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:21.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 24688 24672 140.161 109.188 0.00211515 0.00710287 2024-08-17T17:30:21.796 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:21.817 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:21.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:22.045 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:22.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:22.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:22.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:22.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:22.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:22.349 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:22.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:22 smithi175 ceph-mon[101434]: pgmap v714: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 6211/58758 objects degraded (10.570%); 7/58758 objects misplaced (0.012%) 2024-08-17T17:30:22.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:22 smithi064 ceph-mon[104516]: pgmap v714: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 6211/58758 objects degraded (10.570%); 7/58758 objects misplaced (0.012%) 2024-08-17T17:30:22.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:22 smithi064 ceph-mon[109246]: pgmap v714: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 6211/58758 objects degraded (10.570%); 7/58758 objects misplaced (0.012%) 2024-08-17T17:30:22.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:22.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:22.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:22.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:22.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:22.762 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:22.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:22.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 26366 26350 137.22 104.875 0.0106617 0.00728237 2024-08-17T17:30:22.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:22.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:22.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:22.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:23.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:23.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:23.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 6758/62097 objects degraded (10.883%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:23.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 6758/62097 objects degraded (10.883%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:23.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:23.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:23.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:23.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:23.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.607 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:23.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 6758/62097 objects degraded (10.883%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:23.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:23.731 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:23.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 28318 28302 136.048 122 0.00777762 0.00734424 2024-08-17T17:30:23.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:23.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.871 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:23.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:23.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:24.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:24.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:24.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:24.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:24.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:24.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:24.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:24 smithi064 ceph-mon[104516]: pgmap v715: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 6758/62097 objects degraded (10.883%); 7/62097 objects misplaced (0.011%) 2024-08-17T17:30:24.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:23] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:30:24.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:24 smithi064 ceph-mon[109246]: pgmap v715: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 6758/62097 objects degraded (10.883%); 7/62097 objects misplaced (0.011%) 2024-08-17T17:30:24.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.552 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:24.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:24.653 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:24 smithi175 ceph-mon[101434]: pgmap v715: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 6758/62097 objects degraded (10.883%); 7/62097 objects misplaced (0.011%) 2024-08-17T17:30:24.677 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.775 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 30304 30288 135.196 124.125 0.00878269 0.00739105 2024-08-17T17:30:24.778 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:24.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:25.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:25.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:25.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:25.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:25.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:25.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:25.433 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:25.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:25.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:25.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:25.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:25.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:25.617 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:25.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:25.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:25.776 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 32114 32098 133.716 113.125 0.0135809 0.00747283 2024-08-17T17:30:25.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:25.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:25.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:25.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:26.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:26.077 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:26.078 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:26.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:26.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:26.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:26 smithi064 ceph-mon[104516]: pgmap v716: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8965/83493 objects degraded (10.737%); 7/83493 objects misplaced (0.008%) 2024-08-17T17:30:26.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:26 smithi064 ceph-mon[109246]: pgmap v716: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8965/83493 objects degraded (10.737%); 7/83493 objects misplaced (0.008%) 2024-08-17T17:30:26.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:26.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:26.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:26 smithi175 ceph-mon[101434]: pgmap v716: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8965/83493 objects degraded (10.737%); 7/83493 objects misplaced (0.008%) 2024-08-17T17:30:26.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:26.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:26.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:26.776 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 33735 33719 131.69 101.312 0.00751983 0.00758736 2024-08-17T17:30:26.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:26.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:26.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:26.990 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:30:26.991 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:30:27.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:27.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:27.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.189 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:30:27.232 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:27.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1668369178' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:27 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:27.497 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:27.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1668369178' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:27 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1668369178' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:27 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:27.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:30:27.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.655 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:27.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:27.757 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:27.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 35511 35495 130.468 111 0.00547151 0.00765804 2024-08-17T17:30:27.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:27.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:27.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:27.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:28.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:28.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:28.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:28.193 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:28.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:28.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: from='client.45221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: pgmap v717: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 9507/88575 objects degraded (10.733%); 7/88575 objects misplaced (0.008%) 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: osdmap e234: 8 total, 7 up, 8 in 2024-08-17T17:30:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9507/88575 objects degraded (10.733%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:28.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:28.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:28.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: from='client.45221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:28.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: pgmap v717: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 9507/88575 objects degraded (10.733%); 7/88575 objects misplaced (0.008%) 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: osdmap e234: 8 total, 7 up, 8 in 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9507/88575 objects degraded (10.733%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: from='client.45221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: pgmap v717: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 9507/88575 objects degraded (10.733%); 7/88575 objects misplaced (0.008%) 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: osdmap e234: 8 total, 7 up, 8 in 2024-08-17T17:30:28.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9507/88575 objects degraded (10.733%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:28.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:28.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:28.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:28.746 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:28.776 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 15 37335 37320 129.556 114.062 0.0100482 0.00771353 2024-08-17T17:30:28.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:28.859 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:28.871 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:28.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:28.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:29.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:29.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:29.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:29.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:29.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:29.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:29.367 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:29.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:29.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:29.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:29.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:29.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:29.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:29.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:29.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 38907 38891 127.905 98.1875 0.00367659 0.0078083 2024-08-17T17:30:29.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:29.899 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:29.925 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:30.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:30.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:30.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:30.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:30.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:30.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:30:30.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:30.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:30.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:30.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:30.584 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:30 smithi175 ceph-mon[101434]: pgmap v719: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10051/100434 objects degraded (10.008%); 7/100434 objects misplaced (0.007%) 2024-08-17T17:30:30.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:30 smithi064 ceph-mon[104516]: pgmap v719: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10051/100434 objects degraded (10.008%); 7/100434 objects misplaced (0.007%) 2024-08-17T17:30:30.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:30 smithi064 ceph-mon[109246]: pgmap v719: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 10051/100434 objects degraded (10.008%); 7/100434 objects misplaced (0.007%) 2024-08-17T17:30:30.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:30.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:30:30.776974+0000 min lat: 0.00120552 max lat: 0.359763 avg lat: 0.0079246 2024-08-17T17:30:30.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:30:30.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40377 40361 126.103 91.875 0.00648879 0.0079246 2024-08-17T17:30:30.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:30.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:30.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:31.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:31.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:31.142 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:31.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:31.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:31.387 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:31.411 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:31.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:31.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:31.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:31.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:31.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 41768 41752 124.238 86.9375 0.00321913 0.00802883 2024-08-17T17:30:31.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:31.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:31.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:31.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:32.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:32.169 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:32.297 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:32.323 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:32.425 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:32.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:32.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:32 smithi175 ceph-mon[101434]: pgmap v720: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11677/111801 objects degraded (10.444%); 7/111801 objects misplaced (0.006%) 2024-08-17T17:30:32.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:32 smithi064 ceph-mon[104516]: pgmap v720: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11677/111801 objects degraded (10.444%); 7/111801 objects misplaced (0.006%) 2024-08-17T17:30:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:32 smithi064 ceph-mon[109246]: pgmap v720: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11677/111801 objects degraded (10.444%); 7/111801 objects misplaced (0.006%) 2024-08-17T17:30:32.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:32.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 43173 43157 122.582 87.8125 0.00398432 0.00815186 2024-08-17T17:30:32.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:32.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:32.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:32.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.083 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:33.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:33.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:33.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:33.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:33.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:33.480 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:33.504 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 11677/111801 objects degraded (10.444%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:33.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 11677/111801 objects degraded (10.444%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:33.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:33.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:33.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 11677/111801 objects degraded (10.444%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:33.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:33.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:33.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 44837 44821 121.774 104 0.00794077 0.00820665 2024-08-17T17:30:33.836 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":716,"stamp":"2024-08-17T17:30:24.985862+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1826887653,"num_objects":27831,"num_object_clones":0,"num_object_copies":83493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8965,"num_objects_misplaced":7,"num_objects_unfound":0,"num_objects_dirty":27831,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":28926,"num_write_kb":1794433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":10166083,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28985,"ondisk_log_size":28985,"up":235,"acting":244,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":289,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9941672,"kb_used_data":7124968,"kb_used_omap":34,"kb_used_meta":2816605,"kb_avail":740019544,"statfs":{"total":767960285184,"available":757780013056,"internally_reserved":0,"allocated":7295967232,"data_stored":7286081674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35823,"internal_metadata":2884203537},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1808007168,"num_objects":27588,"num_object_clones":0,"num_object_copies":82764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27588,"num_write_kb":1765632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004598"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":322,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008438+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:30:09.008438+0000","last_peered":"2024-08-17T17:30:09.008438+0000","last_clean":"2024-08-17T17:30:09.008438+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:30:09.008438+0000","last_undegraded":"2024-08-17T17:30:09.008438+0000","last_fullsized":"2024-08-17T17:30:09.008438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":229,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.640656+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:30:07.640656+0000","last_peered":"2024-08-17T17:30:07.640656+0000","last_clean":"2024-08-17T17:30:07.640656+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:30:07.640656+0000","last_undegraded":"2024-08-17T17:30:07.640656+0000","last_fullsized":"2024-08-17T17:30:07.640656+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":26,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163160+0000","last_change":"2024-08-17T17:29:17.304977+0000","last_active":"2024-08-17T17:30:08.163160+0000","last_peered":"2024-08-17T17:30:08.163160+0000","last_clean":"2024-08-17T17:30:08.163160+0000","last_became_active":"2024-08-17T17:29:17.304697+0000","last_became_peered":"2024-08-17T17:29:17.304697+0000","last_unstale":"2024-08-17T17:30:08.163160+0000","last_undegraded":"2024-08-17T17:30:08.163160+0000","last_fullsized":"2024-08-17T17:30:08.163160+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:12.146164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":314,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784134+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:30:08.784134+0000","last_peered":"2024-08-17T17:30:08.784134+0000","last_clean":"2024-08-17T17:30:08.784134+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:30:08.784134+0000","last_undegraded":"2024-08-17T17:30:08.784134+0000","last_fullsized":"2024-08-17T17:30:08.784134+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":314,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784365+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:30:08.784365+0000","last_peered":"2024-08-17T17:30:08.784365+0000","last_clean":"2024-08-17T17:30:08.784365+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:30:08.784365+0000","last_undegraded":"2024-08-17T17:30:08.784365+0000","last_fullsized":"2024-08-17T17:30:08.784365+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:07.640324+0000","last_change":"2024-08-17T17:27:36.155814+0000","last_active":"2024-08-17T17:30:07.640324+0000","last_peered":"2024-08-17T17:30:07.640324+0000","last_clean":"2024-08-17T17:27:34.150951+0000","last_became_active":"2024-08-17T17:27:36.155814+0000","last_became_peered":"2024-08-17T17:27:36.155814+0000","last_unstale":"2024-08-17T17:30:07.640324+0000","last_undegraded":"2024-08-17T17:30:07.640324+0000","last_fullsized":"2024-08-17T17:27:36.151421+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:37.003689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":322,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.009232+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:30:09.009232+0000","last_peered":"2024-08-17T17:30:09.009232+0000","last_clean":"2024-08-17T17:30:09.009232+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:30:09.009232+0000","last_undegraded":"2024-08-17T17:30:09.009232+0000","last_fullsized":"2024-08-17T17:30:09.009232+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":102,"reported_epoch":233,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.783745+0000","last_change":"2024-08-17T17:29:18.607304+0000","last_active":"2024-08-17T17:30:08.783745+0000","last_peered":"2024-08-17T17:30:08.783745+0000","last_clean":"2024-08-17T17:30:08.783745+0000","last_became_active":"2024-08-17T17:29:18.408751+0000","last_became_peered":"2024-08-17T17:29:18.408751+0000","last_unstale":"2024-08-17T17:30:08.783745+0000","last_undegraded":"2024-08-17T17:30:08.783745+0000","last_fullsized":"2024-08-17T17:30:08.783745+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:54.055698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.b","version":"233'1892","reported_seq":1910,"reported_epoch":233,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:24.617029+0000","last_change":"2024-08-17T17:30:10.779089+0000","last_active":"2024-08-17T17:30:24.617029+0000","last_peered":"2024-08-17T17:30:24.617029+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:29:57.199540+0000","last_became_peered":"2024-08-17T17:29:57.199540+0000","last_unstale":"2024-08-17T17:30:24.617029+0000","last_undegraded":"2024-08-17T17:30:10.777663+0000","last_fullsized":"2024-08-17T17:29:57.195434+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1892,"log_dups_size":0,"ondisk_log_size":1892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:58.585593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123994112,"num_objects":1892,"num_object_clones":0,"num_object_copies":5676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1892,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1892,"num_write_kb":121088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1892}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":224,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.640781+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:30:07.640781+0000","last_peered":"2024-08-17T17:30:07.640781+0000","last_clean":"2024-08-17T17:30:07.640781+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:30:07.640781+0000","last_undegraded":"2024-08-17T17:30:07.640781+0000","last_fullsized":"2024-08-17T17:30:07.640781+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.162515+0000","last_change":"2024-08-17T17:29:18.300854+0000","last_active":"2024-08-17T17:30:08.162515+0000","last_peered":"2024-08-17T17:30:08.162515+0000","last_clean":"2024-08-17T17:30:08.162515+0000","last_became_active":"2024-08-17T17:29:18.300442+0000","last_became_peered":"2024-08-17T17:29:18.300442+0000","last_unstale":"2024-08-17T17:30:08.162515+0000","last_undegraded":"2024-08-17T17:30:08.162515+0000","last_fullsized":"2024-08-17T17:30:08.162515+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:10.397667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,4],"avail_no_missing":["5","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"233'1789","reported_seq":1810,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:24.178201+0000","last_change":"2024-08-17T17:29:57.198792+0000","last_active":"2024-08-17T17:30:24.178201+0000","last_peered":"2024-08-17T17:30:24.178201+0000","last_clean":"2024-08-17T17:30:24.178201+0000","last_became_active":"2024-08-17T17:29:57.198502+0000","last_became_peered":"2024-08-17T17:29:57.198502+0000","last_unstale":"2024-08-17T17:30:24.178201+0000","last_undegraded":"2024-08-17T17:30:24.178201+0000","last_fullsized":"2024-08-17T17:30:24.178201+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1789,"log_dups_size":0,"ondisk_log_size":1789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:13.630244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1789,"num_write_kb":114496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":226,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:10.779092+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:30:10.779092+0000","last_peered":"2024-08-17T17:30:10.779092+0000","last_clean":"2024-08-17T17:30:10.779092+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:30:10.779092+0000","last_undegraded":"2024-08-17T17:30:10.779092+0000","last_fullsized":"2024-08-17T17:30:10.779092+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163191+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:30:08.163191+0000","last_peered":"2024-08-17T17:30:08.163191+0000","last_clean":"2024-08-17T17:30:08.163191+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:30:08.163191+0000","last_undegraded":"2024-08-17T17:30:08.163191+0000","last_fullsized":"2024-08-17T17:30:08.163191+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"233'1822","reported_seq":1842,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.730332+0000","last_change":"2024-08-17T17:29:57.202283+0000","last_active":"2024-08-17T17:30:23.730332+0000","last_peered":"2024-08-17T17:30:23.730332+0000","last_clean":"2024-08-17T17:30:23.730332+0000","last_became_active":"2024-08-17T17:29:57.201797+0000","last_became_peered":"2024-08-17T17:29:57.201797+0000","last_unstale":"2024-08-17T17:30:23.730332+0000","last_undegraded":"2024-08-17T17:30:23.730332+0000","last_fullsized":"2024-08-17T17:30:23.730332+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1822,"log_dups_size":0,"ondisk_log_size":1822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:53.663893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119406592,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1822,"num_write_kb":116608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":217,"reported_epoch":233,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:10.778855+0000","last_change":"2024-08-17T17:29:18.306246+0000","last_active":"2024-08-17T17:30:10.778855+0000","last_peered":"2024-08-17T17:30:10.778855+0000","last_clean":"2024-08-17T17:30:10.778855+0000","last_became_active":"2024-08-17T17:29:18.302928+0000","last_became_peered":"2024-08-17T17:29:18.302928+0000","last_unstale":"2024-08-17T17:30:10.778855+0000","last_undegraded":"2024-08-17T17:30:10.778855+0000","last_fullsized":"2024-08-17T17:30:10.778855+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:59.850348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":211,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:10.249216+0000","last_change":"2024-08-17T17:27:36.157622+0000","last_active":"2024-08-17T17:30:10.249216+0000","last_peered":"2024-08-17T17:30:10.249216+0000","last_clean":"2024-08-17T17:26:20.952601+0000","last_became_active":"2024-08-17T17:27:36.157622+0000","last_became_peered":"2024-08-17T17:27:36.157622+0000","last_unstale":"2024-08-17T17:30:10.249216+0000","last_undegraded":"2024-08-17T17:30:10.249216+0000","last_fullsized":"2024-08-17T17:27:36.152425+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:18.183484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.8","version":"233'1665","reported_seq":1686,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.486406+0000","last_change":"2024-08-17T17:29:57.200346+0000","last_active":"2024-08-17T17:30:23.486406+0000","last_peered":"2024-08-17T17:30:23.486406+0000","last_clean":"2024-08-17T17:30:23.486406+0000","last_became_active":"2024-08-17T17:29:57.200066+0000","last_became_peered":"2024-08-17T17:29:57.200066+0000","last_unstale":"2024-08-17T17:30:23.486406+0000","last_undegraded":"2024-08-17T17:30:23.486406+0000","last_fullsized":"2024-08-17T17:30:23.486406+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1665,"log_dups_size":0,"ondisk_log_size":1665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:39.347444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1665,"num_write_kb":106560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":27,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.162871+0000","last_change":"2024-08-17T17:29:18.524198+0000","last_active":"2024-08-17T17:30:08.162871+0000","last_peered":"2024-08-17T17:30:08.162871+0000","last_clean":"2024-08-17T17:30:08.162871+0000","last_became_active":"2024-08-17T17:29:18.523424+0000","last_became_peered":"2024-08-17T17:29:18.523424+0000","last_unstale":"2024-08-17T17:30:08.162871+0000","last_undegraded":"2024-08-17T17:30:08.162871+0000","last_fullsized":"2024-08-17T17:30:08.162871+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:42.296332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,1],"avail_no_missing":["5","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":328,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784140+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:30:08.784140+0000","last_peered":"2024-08-17T17:30:08.784140+0000","last_clean":"2024-08-17T17:30:08.784140+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:30:08.784140+0000","last_undegraded":"2024-08-17T17:30:08.784140+0000","last_fullsized":"2024-08-17T17:30:08.784140+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"233'1816","reported_seq":1837,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.678307+0000","last_change":"2024-08-17T17:29:57.204892+0000","last_active":"2024-08-17T17:30:23.678307+0000","last_peered":"2024-08-17T17:30:23.678307+0000","last_clean":"2024-08-17T17:30:23.678307+0000","last_became_active":"2024-08-17T17:29:57.204591+0000","last_became_peered":"2024-08-17T17:29:57.204591+0000","last_unstale":"2024-08-17T17:30:23.678307+0000","last_undegraded":"2024-08-17T17:30:23.678307+0000","last_fullsized":"2024-08-17T17:30:23.678307+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1816,"log_dups_size":0,"ondisk_log_size":1816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:25.247583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1816,"num_write_kb":116224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":344,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008549+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:30:09.008549+0000","last_peered":"2024-08-17T17:30:09.008549+0000","last_clean":"2024-08-17T17:30:09.008549+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:30:09.008549+0000","last_undegraded":"2024-08-17T17:30:09.008549+0000","last_fullsized":"2024-08-17T17:30:09.008549+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.163416+0000","last_change":"2024-08-17T17:27:36.162164+0000","last_active":"2024-08-17T17:30:08.163416+0000","last_peered":"2024-08-17T17:30:08.163416+0000","last_clean":"2024-08-17T17:27:34.153084+0000","last_became_active":"2024-08-17T17:27:36.162164+0000","last_became_peered":"2024-08-17T17:27:36.162164+0000","last_unstale":"2024-08-17T17:30:08.163416+0000","last_undegraded":"2024-08-17T17:30:08.163416+0000","last_fullsized":"2024-08-17T17:27:36.150716+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:39.809556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"233'1728","reported_seq":1748,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.502440+0000","last_change":"2024-08-17T17:29:57.203806+0000","last_active":"2024-08-17T17:30:23.502440+0000","last_peered":"2024-08-17T17:30:23.502440+0000","last_clean":"2024-08-17T17:30:23.502440+0000","last_became_active":"2024-08-17T17:29:57.203486+0000","last_became_peered":"2024-08-17T17:29:57.203486+0000","last_unstale":"2024-08-17T17:30:23.502440+0000","last_undegraded":"2024-08-17T17:30:23.502440+0000","last_fullsized":"2024-08-17T17:30:23.502440+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1728,"log_dups_size":0,"ondisk_log_size":1728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:36:11.178889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113246208,"num_objects":1728,"num_object_clones":0,"num_object_copies":5184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1728,"num_write_kb":110592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":140,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.163066+0000","last_change":"2024-08-17T17:27:36.173059+0000","last_active":"2024-08-17T17:30:08.163066+0000","last_peered":"2024-08-17T17:30:08.163066+0000","last_clean":"2024-08-17T17:22:13.178082+0000","last_became_active":"2024-08-17T17:27:36.173059+0000","last_became_peered":"2024-08-17T17:27:36.173059+0000","last_unstale":"2024-08-17T17:30:08.163066+0000","last_undegraded":"2024-08-17T17:30:08.163066+0000","last_fullsized":"2024-08-17T17:27:36.150235+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:34.696702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163100+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:30:08.163100+0000","last_peered":"2024-08-17T17:30:08.163100+0000","last_clean":"2024-08-17T17:30:08.163100+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:30:08.163100+0000","last_undegraded":"2024-08-17T17:30:08.163100+0000","last_fullsized":"2024-08-17T17:30:08.163100+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"233'1923","reported_seq":1941,"reported_epoch":233,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:24.624751+0000","last_change":"2024-08-17T17:30:10.786954+0000","last_active":"2024-08-17T17:30:24.624751+0000","last_peered":"2024-08-17T17:30:24.624751+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:29:57.202642+0000","last_became_peered":"2024-08-17T17:29:57.202642+0000","last_unstale":"2024-08-17T17:30:24.624751+0000","last_undegraded":"2024-08-17T17:30:10.784317+0000","last_fullsized":"2024-08-17T17:29:57.195072+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1923,"log_dups_size":0,"ondisk_log_size":1923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:02:07.038249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126025728,"num_objects":1923,"num_object_clones":0,"num_object_copies":5769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1923,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":123072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1923}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163048+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:30:08.163048+0000","last_peered":"2024-08-17T17:30:08.163048+0000","last_clean":"2024-08-17T17:30:08.163048+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:30:08.163048+0000","last_undegraded":"2024-08-17T17:30:08.163048+0000","last_fullsized":"2024-08-17T17:30:08.163048+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.163448+0000","last_change":"2024-08-17T17:27:36.176161+0000","last_active":"2024-08-17T17:30:08.163448+0000","last_peered":"2024-08-17T17:30:08.163448+0000","last_clean":"2024-08-17T17:27:34.153378+0000","last_became_active":"2024-08-17T17:27:36.176161+0000","last_became_peered":"2024-08-17T17:27:36.176161+0000","last_unstale":"2024-08-17T17:30:08.163448+0000","last_undegraded":"2024-08-17T17:30:08.163448+0000","last_fullsized":"2024-08-17T17:27:36.151362+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:06.264054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"233'1790","reported_seq":1808,"reported_epoch":233,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:24.182507+0000","last_change":"2024-08-17T17:30:10.782470+0000","last_active":"2024-08-17T17:30:24.182507+0000","last_peered":"2024-08-17T17:30:24.182507+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:29:57.199293+0000","last_became_peered":"2024-08-17T17:29:57.199293+0000","last_unstale":"2024-08-17T17:30:24.182507+0000","last_undegraded":"2024-08-17T17:30:10.781649+0000","last_fullsized":"2024-08-17T17:29:57.192135+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1790,"log_dups_size":0,"ondisk_log_size":1790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:35.087797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1790,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1790,"num_write_kb":114560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1790}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":324,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008445+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:30:09.008445+0000","last_peered":"2024-08-17T17:30:09.008445+0000","last_clean":"2024-08-17T17:30:09.008445+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:30:09.008445+0000","last_undegraded":"2024-08-17T17:30:09.008445+0000","last_fullsized":"2024-08-17T17:30:09.008445+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":311,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:10.249560+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:30:10.249560+0000","last_peered":"2024-08-17T17:30:10.249560+0000","last_clean":"2024-08-17T17:30:10.249560+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:30:10.249560+0000","last_undegraded":"2024-08-17T17:30:10.249560+0000","last_fullsized":"2024-08-17T17:30:10.249560+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"233'1757","reported_seq":1778,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.678965+0000","last_change":"2024-08-17T17:29:57.204368+0000","last_active":"2024-08-17T17:30:23.678965+0000","last_peered":"2024-08-17T17:30:23.678965+0000","last_clean":"2024-08-17T17:30:23.678965+0000","last_became_active":"2024-08-17T17:29:57.204070+0000","last_became_peered":"2024-08-17T17:29:57.204070+0000","last_unstale":"2024-08-17T17:30:23.678965+0000","last_undegraded":"2024-08-17T17:30:23.678965+0000","last_fullsized":"2024-08-17T17:30:23.678965+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1757,"log_dups_size":0,"ondisk_log_size":1757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:00.628676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115146752,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1757,"num_write_kb":112448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":536,"reported_epoch":232,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:07.640276+0000","last_change":"2024-08-17T17:27:36.158454+0000","last_active":"2024-08-17T17:30:07.640276+0000","last_peered":"2024-08-17T17:30:07.640276+0000","last_clean":"2024-08-17T17:27:34.151184+0000","last_became_active":"2024-08-17T17:27:36.158454+0000","last_became_peered":"2024-08-17T17:27:36.158454+0000","last_unstale":"2024-08-17T17:30:07.640276+0000","last_undegraded":"2024-08-17T17:27:36.152383+0000","last_fullsized":"2024-08-17T17:27:36.152283+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:54.606297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":831,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.162695+0000","last_change":"2024-08-17T17:29:20.347394+0000","last_active":"2024-08-17T17:30:08.162695+0000","last_peered":"2024-08-17T17:30:08.162695+0000","last_clean":"2024-08-17T17:30:08.162695+0000","last_became_active":"2024-08-17T17:29:20.346944+0000","last_became_peered":"2024-08-17T17:29:20.346944+0000","last_unstale":"2024-08-17T17:30:08.162695+0000","last_undegraded":"2024-08-17T17:30:08.162695+0000","last_fullsized":"2024-08-17T17:30:08.162695+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:58.904569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":291,"reported_epoch":233,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:09.008594+0000","last_change":"2024-08-17T17:29:18.309474+0000","last_active":"2024-08-17T17:30:09.008594+0000","last_peered":"2024-08-17T17:30:09.008594+0000","last_clean":"2024-08-17T17:30:09.008594+0000","last_became_active":"2024-08-17T17:29:18.307914+0000","last_became_peered":"2024-08-17T17:29:18.307914+0000","last_unstale":"2024-08-17T17:30:09.008594+0000","last_undegraded":"2024-08-17T17:30:09.008594+0000","last_fullsized":"2024-08-17T17:30:09.008594+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:17:48.393248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,4],"avail_no_missing":["3","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":124,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.162490+0000","last_change":"2024-08-17T17:27:36.153705+0000","last_active":"2024-08-17T17:30:08.162490+0000","last_peered":"2024-08-17T17:30:08.162490+0000","last_clean":"2024-08-17T17:22:13.179329+0000","last_became_active":"2024-08-17T17:27:36.153705+0000","last_became_peered":"2024-08-17T17:27:36.153705+0000","last_unstale":"2024-08-17T17:30:08.162490+0000","last_undegraded":"2024-08-17T17:30:08.162490+0000","last_fullsized":"2024-08-17T17:27:36.150850+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:19.689789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"233'1449","reported_seq":1463,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:21.072436+0000","last_change":"2024-08-17T17:29:57.203033+0000","last_active":"2024-08-17T17:30:21.072436+0000","last_peered":"2024-08-17T17:30:21.072436+0000","last_clean":"2024-08-17T17:30:21.072436+0000","last_became_active":"2024-08-17T17:29:57.202766+0000","last_became_peered":"2024-08-17T17:29:57.202766+0000","last_unstale":"2024-08-17T17:30:21.072436+0000","last_undegraded":"2024-08-17T17:30:21.072436+0000","last_fullsized":"2024-08-17T17:30:21.072436+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1449,"log_dups_size":0,"ondisk_log_size":1449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:06.312698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94961664,"num_objects":1449,"num_object_clones":0,"num_object_copies":4347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1449,"num_write_kb":92736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":607,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.640422+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:30:07.640422+0000","last_peered":"2024-08-17T17:30:07.640422+0000","last_clean":"2024-08-17T17:30:07.640422+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:30:07.640422+0000","last_undegraded":"2024-08-17T17:30:07.640422+0000","last_fullsized":"2024-08-17T17:30:07.640422+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":329,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.783894+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:30:08.783894+0000","last_peered":"2024-08-17T17:30:08.783894+0000","last_clean":"2024-08-17T17:30:08.783894+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:30:08.783894+0000","last_undegraded":"2024-08-17T17:30:08.783894+0000","last_fullsized":"2024-08-17T17:30:08.783894+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784550+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:30:08.784550+0000","last_peered":"2024-08-17T17:30:08.784550+0000","last_clean":"2024-08-17T17:30:08.784550+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:30:08.784550+0000","last_undegraded":"2024-08-17T17:30:08.784550+0000","last_fullsized":"2024-08-17T17:30:08.784550+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"233'1749","reported_seq":1769,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:23.723317+0000","last_change":"2024-08-17T17:29:57.202172+0000","last_active":"2024-08-17T17:30:23.723317+0000","last_peered":"2024-08-17T17:30:23.723317+0000","last_clean":"2024-08-17T17:30:23.723317+0000","last_became_active":"2024-08-17T17:29:57.201785+0000","last_became_peered":"2024-08-17T17:29:57.201785+0000","last_unstale":"2024-08-17T17:30:23.723317+0000","last_undegraded":"2024-08-17T17:30:23.723317+0000","last_fullsized":"2024-08-17T17:30:23.723317+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1749,"log_dups_size":0,"ondisk_log_size":1749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:28.967156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1749,"num_write_kb":111936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":725,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:10.249233+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:30:10.249233+0000","last_peered":"2024-08-17T17:30:10.249233+0000","last_clean":"2024-08-17T17:30:10.249233+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:30:10.249233+0000","last_undegraded":"2024-08-17T17:30:10.249233+0000","last_fullsized":"2024-08-17T17:30:10.249233+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":310,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.648014+0000","last_change":"2024-08-17T17:27:36.176565+0000","last_active":"2024-08-17T17:30:08.648014+0000","last_peered":"2024-08-17T17:30:08.648014+0000","last_clean":"2024-08-17T17:27:34.150964+0000","last_became_active":"2024-08-17T17:27:36.176565+0000","last_became_peered":"2024-08-17T17:27:36.176565+0000","last_unstale":"2024-08-17T17:30:08.648014+0000","last_undegraded":"2024-08-17T17:30:08.648014+0000","last_fullsized":"2024-08-17T17:27:36.158516+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:38:03.408152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":353,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.648546+0000","last_change":"2024-08-17T17:29:17.310485+0000","last_active":"2024-08-17T17:30:08.648546+0000","last_peered":"2024-08-17T17:30:08.648546+0000","last_clean":"2024-08-17T17:30:08.648546+0000","last_became_active":"2024-08-17T17:29:17.310110+0000","last_became_peered":"2024-08-17T17:29:17.310110+0000","last_unstale":"2024-08-17T17:30:08.648546+0000","last_undegraded":"2024-08-17T17:30:08.648546+0000","last_fullsized":"2024-08-17T17:30:08.648546+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:24.885692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"233'1850","reported_seq":1868,"reported_epoch":233,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:23.723043+0000","last_change":"2024-08-17T17:30:10.797133+0000","last_active":"2024-08-17T17:30:23.723043+0000","last_peered":"2024-08-17T17:30:23.723043+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:29:57.201826+0000","last_became_peered":"2024-08-17T17:29:57.201826+0000","last_unstale":"2024-08-17T17:30:23.723043+0000","last_undegraded":"2024-08-17T17:30:10.777907+0000","last_fullsized":"2024-08-17T17:29:57.197100+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1850,"log_dups_size":0,"ondisk_log_size":1850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:56.463418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121241600,"num_objects":1850,"num_object_clones":0,"num_object_copies":5550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1850,"num_write_kb":118400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1850}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":946,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.163589+0000","last_change":"2024-08-17T17:29:20.361336+0000","last_active":"2024-08-17T17:30:08.163589+0000","last_peered":"2024-08-17T17:30:08.163589+0000","last_clean":"2024-08-17T17:30:08.163589+0000","last_became_active":"2024-08-17T17:29:20.360745+0000","last_became_peered":"2024-08-17T17:29:20.360745+0000","last_unstale":"2024-08-17T17:30:08.163589+0000","last_undegraded":"2024-08-17T17:30:08.163589+0000","last_fullsized":"2024-08-17T17:30:08.163589+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:29.330894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,0],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:10.249309+0000","last_change":"2024-08-17T17:27:36.165070+0000","last_active":"2024-08-17T17:30:10.249309+0000","last_peered":"2024-08-17T17:30:10.249309+0000","last_clean":"2024-08-17T17:27:34.159785+0000","last_became_active":"2024-08-17T17:27:36.165070+0000","last_became_peered":"2024-08-17T17:27:36.165070+0000","last_unstale":"2024-08-17T17:30:10.249309+0000","last_undegraded":"2024-08-17T17:30:10.249309+0000","last_fullsized":"2024-08-17T17:27:36.153509+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:36.585084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":211,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.648312+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:30:08.648312+0000","last_peered":"2024-08-17T17:30:08.648312+0000","last_clean":"2024-08-17T17:30:08.648312+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:30:08.648312+0000","last_undegraded":"2024-08-17T17:30:08.648312+0000","last_fullsized":"2024-08-17T17:30:08.648312+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"233'1504","reported_seq":1515,"reported_epoch":233,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:21.576362+0000","last_change":"2024-08-17T17:30:10.788218+0000","last_active":"2024-08-17T17:30:21.576362+0000","last_peered":"2024-08-17T17:30:21.576362+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:29:57.200360+0000","last_became_peered":"2024-08-17T17:29:57.200360+0000","last_unstale":"2024-08-17T17:30:21.576362+0000","last_undegraded":"2024-08-17T17:30:10.779101+0000","last_fullsized":"2024-08-17T17:29:57.194930+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1504,"log_dups_size":0,"ondisk_log_size":1504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:33:25.922263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98566144,"num_objects":1504,"num_object_clones":0,"num_object_copies":4512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1504,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1504,"num_write_kb":96256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":1504}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":26,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163035+0000","last_change":"2024-08-17T17:29:17.307600+0000","last_active":"2024-08-17T17:30:08.163035+0000","last_peered":"2024-08-17T17:30:08.163035+0000","last_clean":"2024-08-17T17:30:08.163035+0000","last_became_active":"2024-08-17T17:29:17.307017+0000","last_became_peered":"2024-08-17T17:29:17.307017+0000","last_unstale":"2024-08-17T17:30:08.163035+0000","last_undegraded":"2024-08-17T17:30:08.163035+0000","last_fullsized":"2024-08-17T17:30:08.163035+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:36:07.327855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":610,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.648457+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:30:08.648457+0000","last_peered":"2024-08-17T17:30:08.648457+0000","last_clean":"2024-08-17T17:30:08.648457+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:30:08.648457+0000","last_undegraded":"2024-08-17T17:30:08.648457+0000","last_fullsized":"2024-08-17T17:30:08.648457+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":323,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.648213+0000","last_change":"2024-08-17T17:29:17.310523+0000","last_active":"2024-08-17T17:30:08.648213+0000","last_peered":"2024-08-17T17:30:08.648213+0000","last_clean":"2024-08-17T17:30:08.648213+0000","last_became_active":"2024-08-17T17:29:17.310078+0000","last_became_peered":"2024-08-17T17:29:17.310078+0000","last_unstale":"2024-08-17T17:30:08.648213+0000","last_undegraded":"2024-08-17T17:30:08.648213+0000","last_fullsized":"2024-08-17T17:30:08.648213+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:40.618055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"233'1434","reported_seq":1447,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:20.841065+0000","last_change":"2024-08-17T17:29:57.200752+0000","last_active":"2024-08-17T17:30:20.841065+0000","last_peered":"2024-08-17T17:30:20.841065+0000","last_clean":"2024-08-17T17:30:20.841065+0000","last_became_active":"2024-08-17T17:29:57.200431+0000","last_became_peered":"2024-08-17T17:29:57.200431+0000","last_unstale":"2024-08-17T17:30:20.841065+0000","last_undegraded":"2024-08-17T17:30:20.841065+0000","last_fullsized":"2024-08-17T17:30:20.841065+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1434,"log_dups_size":0,"ondisk_log_size":1434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:15.488964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93978624,"num_objects":1434,"num_object_clones":0,"num_object_copies":4302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1434,"num_write_kb":91776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":320,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.648428+0000","last_change":"2024-08-17T17:29:17.310414+0000","last_active":"2024-08-17T17:30:08.648428+0000","last_peered":"2024-08-17T17:30:08.648428+0000","last_clean":"2024-08-17T17:30:08.648428+0000","last_became_active":"2024-08-17T17:29:17.309934+0000","last_became_peered":"2024-08-17T17:29:17.309934+0000","last_unstale":"2024-08-17T17:30:08.648428+0000","last_undegraded":"2024-08-17T17:30:08.648428+0000","last_fullsized":"2024-08-17T17:30:08.648428+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:42.750524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":797,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.640746+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:30:07.640746+0000","last_peered":"2024-08-17T17:30:07.640746+0000","last_clean":"2024-08-17T17:30:07.640746+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:30:07.640746+0000","last_undegraded":"2024-08-17T17:30:07.640746+0000","last_fullsized":"2024-08-17T17:30:07.640746+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":318,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784096+0000","last_change":"2024-08-17T17:29:17.304078+0000","last_active":"2024-08-17T17:30:08.784096+0000","last_peered":"2024-08-17T17:30:08.784096+0000","last_clean":"2024-08-17T17:30:08.784096+0000","last_became_active":"2024-08-17T17:29:17.303528+0000","last_became_peered":"2024-08-17T17:29:17.303528+0000","last_unstale":"2024-08-17T17:30:08.784096+0000","last_undegraded":"2024-08-17T17:30:08.784096+0000","last_fullsized":"2024-08-17T17:30:08.784096+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:42.523997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"233'1801","reported_seq":1822,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:24.180194+0000","last_change":"2024-08-17T17:29:57.198724+0000","last_active":"2024-08-17T17:30:24.180194+0000","last_peered":"2024-08-17T17:30:24.180194+0000","last_clean":"2024-08-17T17:30:24.180194+0000","last_became_active":"2024-08-17T17:29:57.198432+0000","last_became_peered":"2024-08-17T17:29:57.198432+0000","last_unstale":"2024-08-17T17:30:24.180194+0000","last_undegraded":"2024-08-17T17:30:24.180194+0000","last_fullsized":"2024-08-17T17:30:24.180194+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1801,"log_dups_size":0,"ondisk_log_size":1801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:10.292012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118030336,"num_objects":1801,"num_object_clones":0,"num_object_copies":5403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1801,"num_write_kb":115264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":322,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.009300+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:30:09.009300+0000","last_peered":"2024-08-17T17:30:09.009300+0000","last_clean":"2024-08-17T17:30:09.009300+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:30:09.009300+0000","last_undegraded":"2024-08-17T17:30:09.009300+0000","last_fullsized":"2024-08-17T17:30:09.009300+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":833,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.648723+0000","last_change":"2024-08-17T17:27:36.171574+0000","last_active":"2024-08-17T17:30:08.648723+0000","last_peered":"2024-08-17T17:30:08.648723+0000","last_clean":"2024-08-17T17:27:34.151904+0000","last_became_active":"2024-08-17T17:27:36.171574+0000","last_became_peered":"2024-08-17T17:27:36.171574+0000","last_unstale":"2024-08-17T17:30:08.648723+0000","last_undegraded":"2024-08-17T17:30:08.648723+0000","last_fullsized":"2024-08-17T17:27:36.158328+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:56.371461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":319,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008166+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:30:09.008166+0000","last_peered":"2024-08-17T17:30:09.008166+0000","last_clean":"2024-08-17T17:30:09.008166+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:30:09.008166+0000","last_undegraded":"2024-08-17T17:30:09.008166+0000","last_fullsized":"2024-08-17T17:30:09.008166+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"233'1834","reported_seq":1854,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:24.166462+0000","last_change":"2024-08-17T17:29:57.199611+0000","last_active":"2024-08-17T17:30:24.166462+0000","last_peered":"2024-08-17T17:30:24.166462+0000","last_clean":"2024-08-17T17:30:24.166462+0000","last_became_active":"2024-08-17T17:29:57.199174+0000","last_became_peered":"2024-08-17T17:29:57.199174+0000","last_unstale":"2024-08-17T17:30:24.166462+0000","last_undegraded":"2024-08-17T17:30:24.166462+0000","last_fullsized":"2024-08-17T17:30:24.166462+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":1834,"log_dups_size":0,"ondisk_log_size":1834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:39.273888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120193024,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1834,"num_write_kb":117376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":341,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784306+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:30:08.784306+0000","last_peered":"2024-08-17T17:30:08.784306+0000","last_clean":"2024-08-17T17:30:08.784306+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:30:08.784306+0000","last_undegraded":"2024-08-17T17:30:08.784306+0000","last_fullsized":"2024-08-17T17:30:08.784306+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":344,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:07.640779+0000","last_change":"2024-08-17T17:27:36.160014+0000","last_active":"2024-08-17T17:30:07.640779+0000","last_peered":"2024-08-17T17:30:07.640779+0000","last_clean":"2024-08-17T17:07:02.886086+0000","last_became_active":"2024-08-17T17:27:36.160014+0000","last_became_peered":"2024-08-17T17:27:36.160014+0000","last_unstale":"2024-08-17T17:30:07.640779+0000","last_undegraded":"2024-08-17T17:30:07.640779+0000","last_fullsized":"2024-08-17T17:27:36.153838+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:12.687947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":144,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.163339+0000","last_change":"2024-08-17T17:29:18.301452+0000","last_active":"2024-08-17T17:30:08.163339+0000","last_peered":"2024-08-17T17:30:08.163339+0000","last_clean":"2024-08-17T17:30:08.163339+0000","last_became_active":"2024-08-17T17:29:18.301065+0000","last_became_peered":"2024-08-17T17:29:18.301065+0000","last_unstale":"2024-08-17T17:30:08.163339+0000","last_undegraded":"2024-08-17T17:30:08.163339+0000","last_fullsized":"2024-08-17T17:30:08.163339+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:07.561471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,0],"avail_no_missing":["4","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":125,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.163505+0000","last_change":"2024-08-17T17:27:36.166803+0000","last_active":"2024-08-17T17:30:08.163505+0000","last_peered":"2024-08-17T17:30:08.163505+0000","last_clean":"2024-08-17T17:27:34.153014+0000","last_became_active":"2024-08-17T17:27:36.166803+0000","last_became_peered":"2024-08-17T17:27:36.166803+0000","last_unstale":"2024-08-17T17:30:08.163505+0000","last_undegraded":"2024-08-17T17:30:08.163505+0000","last_fullsized":"2024-08-17T17:27:36.150441+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:36:39.160270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":135,"reported_epoch":232,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:30:08.162510+0000","last_change":"2024-08-17T17:27:36.153883+0000","last_active":"2024-08-17T17:30:08.162510+0000","last_peered":"2024-08-17T17:30:08.162510+0000","last_clean":"2024-08-17T17:27:34.152919+0000","last_became_active":"2024-08-17T17:27:36.153883+0000","last_became_peered":"2024-08-17T17:27:36.153883+0000","last_unstale":"2024-08-17T17:30:08.162510+0000","last_undegraded":"2024-08-17T17:27:36.150733+0000","last_fullsized":"2024-08-17T17:27:36.150628+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:17.872426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":309,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.648051+0000","last_change":"2024-08-17T17:27:36.176571+0000","last_active":"2024-08-17T17:30:08.648051+0000","last_peered":"2024-08-17T17:30:08.648051+0000","last_clean":"2024-08-17T17:27:34.150999+0000","last_became_active":"2024-08-17T17:27:36.176571+0000","last_became_peered":"2024-08-17T17:27:36.176571+0000","last_unstale":"2024-08-17T17:30:08.648051+0000","last_undegraded":"2024-08-17T17:30:08.648051+0000","last_fullsized":"2024-08-17T17:27:36.172432+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:35.412782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":230,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.640468+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:30:07.640468+0000","last_peered":"2024-08-17T17:30:07.640468+0000","last_clean":"2024-08-17T17:30:07.640468+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:30:07.640468+0000","last_undegraded":"2024-08-17T17:30:07.640468+0000","last_fullsized":"2024-08-17T17:30:07.640468+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":26,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.162765+0000","last_change":"2024-08-17T17:29:17.308165+0000","last_active":"2024-08-17T17:30:08.162765+0000","last_peered":"2024-08-17T17:30:08.162765+0000","last_clean":"2024-08-17T17:30:08.162765+0000","last_became_active":"2024-08-17T17:29:17.307802+0000","last_became_peered":"2024-08-17T17:29:17.307802+0000","last_unstale":"2024-08-17T17:30:08.162765+0000","last_undegraded":"2024-08-17T17:30:08.162765+0000","last_fullsized":"2024-08-17T17:30:08.162765+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:14.902914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":277,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:09.008140+0000","last_change":"2024-08-17T17:27:36.163742+0000","last_active":"2024-08-17T17:30:09.008140+0000","last_peered":"2024-08-17T17:30:09.008140+0000","last_clean":"2024-08-17T17:27:34.154361+0000","last_became_active":"2024-08-17T17:27:36.163742+0000","last_became_peered":"2024-08-17T17:27:36.163742+0000","last_unstale":"2024-08-17T17:30:09.008140+0000","last_undegraded":"2024-08-17T17:30:09.008140+0000","last_fullsized":"2024-08-17T17:27:36.153948+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:33:26.610976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":332,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784517+0000","last_change":"2024-08-17T17:29:17.305798+0000","last_active":"2024-08-17T17:30:08.784517+0000","last_peered":"2024-08-17T17:30:08.784517+0000","last_clean":"2024-08-17T17:30:08.784517+0000","last_became_active":"2024-08-17T17:29:17.305473+0000","last_became_peered":"2024-08-17T17:29:17.305473+0000","last_unstale":"2024-08-17T17:30:08.784517+0000","last_undegraded":"2024-08-17T17:30:08.784517+0000","last_fullsized":"2024-08-17T17:30:08.784517+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:07:54.671044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":326,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:09.008161+0000","last_change":"2024-08-17T17:27:36.164472+0000","last_active":"2024-08-17T17:30:09.008161+0000","last_peered":"2024-08-17T17:30:09.008161+0000","last_clean":"2024-08-17T17:27:34.153209+0000","last_became_active":"2024-08-17T17:27:36.164472+0000","last_became_peered":"2024-08-17T17:27:36.164472+0000","last_unstale":"2024-08-17T17:30:09.008161+0000","last_undegraded":"2024-08-17T17:30:09.008161+0000","last_fullsized":"2024-08-17T17:27:36.155839+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:11.077063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":27,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.162901+0000","last_change":"2024-08-17T17:29:18.460469+0000","last_active":"2024-08-17T17:30:08.162901+0000","last_peered":"2024-08-17T17:30:08.162901+0000","last_clean":"2024-08-17T17:30:08.162901+0000","last_became_active":"2024-08-17T17:29:18.459312+0000","last_became_peered":"2024-08-17T17:29:18.459312+0000","last_unstale":"2024-08-17T17:30:08.162901+0000","last_undegraded":"2024-08-17T17:30:08.162901+0000","last_fullsized":"2024-08-17T17:30:08.162901+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:07.114492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,0],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":338,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008378+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:30:09.008378+0000","last_peered":"2024-08-17T17:30:09.008378+0000","last_clean":"2024-08-17T17:30:09.008378+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:30:09.008378+0000","last_undegraded":"2024-08-17T17:30:09.008378+0000","last_fullsized":"2024-08-17T17:30:09.008378+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":315,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784389+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:30:08.784389+0000","last_peered":"2024-08-17T17:30:08.784389+0000","last_clean":"2024-08-17T17:30:08.784389+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:30:08.784389+0000","last_undegraded":"2024-08-17T17:30:08.784389+0000","last_fullsized":"2024-08-17T17:30:08.784389+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":319,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.647988+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:30:08.647988+0000","last_peered":"2024-08-17T17:30:08.647988+0000","last_clean":"2024-08-17T17:30:08.647988+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:30:08.647988+0000","last_undegraded":"2024-08-17T17:30:08.647988+0000","last_fullsized":"2024-08-17T17:30:08.647988+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":335,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008978+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:30:09.008978+0000","last_peered":"2024-08-17T17:30:09.008978+0000","last_clean":"2024-08-17T17:30:09.008978+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:30:09.008978+0000","last_undegraded":"2024-08-17T17:30:09.008978+0000","last_fullsized":"2024-08-17T17:30:09.008978+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163293+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:30:08.163293+0000","last_peered":"2024-08-17T17:30:08.163293+0000","last_clean":"2024-08-17T17:30:08.163293+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:30:08.163293+0000","last_undegraded":"2024-08-17T17:30:08.163293+0000","last_fullsized":"2024-08-17T17:30:08.163293+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":343,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:10.249379+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:30:10.249379+0000","last_peered":"2024-08-17T17:30:10.249379+0000","last_clean":"2024-08-17T17:30:10.249379+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:30:10.249379+0000","last_undegraded":"2024-08-17T17:30:10.249379+0000","last_fullsized":"2024-08-17T17:30:10.249379+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":321,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784021+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:30:08.784021+0000","last_peered":"2024-08-17T17:30:08.784021+0000","last_clean":"2024-08-17T17:30:08.784021+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:30:08.784021+0000","last_undegraded":"2024-08-17T17:30:08.784021+0000","last_fullsized":"2024-08-17T17:30:08.784021+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163402+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:30:08.163402+0000","last_peered":"2024-08-17T17:30:08.163402+0000","last_clean":"2024-08-17T17:30:08.163402+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:30:08.163402+0000","last_undegraded":"2024-08-17T17:30:08.163402+0000","last_fullsized":"2024-08-17T17:30:08.163402+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":30,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.162774+0000","last_change":"2024-08-17T17:29:17.312370+0000","last_active":"2024-08-17T17:30:08.162774+0000","last_peered":"2024-08-17T17:30:08.162774+0000","last_clean":"2024-08-17T17:30:08.162774+0000","last_became_active":"2024-08-17T17:29:17.307694+0000","last_became_peered":"2024-08-17T17:29:17.307694+0000","last_unstale":"2024-08-17T17:30:08.162774+0000","last_undegraded":"2024-08-17T17:30:08.162774+0000","last_fullsized":"2024-08-17T17:30:08.162774+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:28.982241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.163187+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:30:08.163187+0000","last_peered":"2024-08-17T17:30:08.163187+0000","last_clean":"2024-08-17T17:30:08.163187+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:30:08.163187+0000","last_undegraded":"2024-08-17T17:30:08.163187+0000","last_fullsized":"2024-08-17T17:30:08.163187+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":31,"reported_epoch":232,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:30:08.163124+0000","last_change":"2024-08-17T17:29:18.318077+0000","last_active":"2024-08-17T17:30:08.163124+0000","last_peered":"2024-08-17T17:30:08.163124+0000","last_clean":"2024-08-17T17:30:08.163124+0000","last_became_active":"2024-08-17T17:29:18.310918+0000","last_became_peered":"2024-08-17T17:29:18.310918+0000","last_unstale":"2024-08-17T17:30:08.163124+0000","last_undegraded":"2024-08-17T17:30:08.163124+0000","last_fullsized":"2024-08-17T17:30:08.163124+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:06:19.236031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":340,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.008432+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:30:09.008432+0000","last_peered":"2024-08-17T17:30:09.008432+0000","last_clean":"2024-08-17T17:30:09.008432+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:30:09.008432+0000","last_undegraded":"2024-08-17T17:30:09.008432+0000","last_fullsized":"2024-08-17T17:30:09.008432+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":333,"reported_epoch":233,"state":"active+undersized","last_fresh":"2024-08-17T17:30:09.009162+0000","last_change":"2024-08-17T17:27:36.160202+0000","last_active":"2024-08-17T17:30:09.009162+0000","last_peered":"2024-08-17T17:30:09.009162+0000","last_clean":"2024-08-17T17:27:34.154748+0000","last_became_active":"2024-08-17T17:27:36.160202+0000","last_became_peered":"2024-08-17T17:27:36.160202+0000","last_unstale":"2024-08-17T17:30:09.009162+0000","last_undegraded":"2024-08-17T17:30:09.009162+0000","last_fullsized":"2024-08-17T17:27:36.155435+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:35.895991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":125,"reported_epoch":232,"state":"active+undersized","last_fresh":"2024-08-17T17:30:08.162631+0000","last_change":"2024-08-17T17:27:36.156351+0000","last_active":"2024-08-17T17:30:08.162631+0000","last_peered":"2024-08-17T17:30:08.162631+0000","last_clean":"2024-08-17T17:27:34.152341+0000","last_became_active":"2024-08-17T17:27:36.156351+0000","last_became_peered":"2024-08-17T17:27:36.156351+0000","last_unstale":"2024-08-17T17:30:08.162631+0000","last_undegraded":"2024-08-17T17:30:08.162631+0000","last_fullsized":"2024-08-17T17:27:36.149941+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:07.514027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":349,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.784463+0000","last_change":"2024-08-17T17:29:17.311568+0000","last_active":"2024-08-17T17:30:08.784463+0000","last_peered":"2024-08-17T17:30:08.784463+0000","last_clean":"2024-08-17T17:30:08.784463+0000","last_became_active":"2024-08-17T17:29:17.306761+0000","last_became_peered":"2024-08-17T17:29:17.306761+0000","last_unstale":"2024-08-17T17:30:08.784463+0000","last_undegraded":"2024-08-17T17:30:08.784463+0000","last_fullsized":"2024-08-17T17:30:08.784463+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:15:11.174870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1822097408,"num_objects":27803,"num_object_clones":0,"num_object_copies":83409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27803,"num_write_kb":1779392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4782686208,"data_stored":4782686208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27803,"ondisk_log_size":27803,"up":43,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17830,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":85,"acting":90,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":21,"acting":22,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":2,"acting":2,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201602,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":798796,"kb_used_data":438724,"kb_used_omap":1,"kb_used_meta":360062,"kb_avail":92946356,"statfs":{"total":95995035648,"available":95177068544,"internally_reserved":0,"allocated":449253376,"data_stored":447979955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":368703947},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463206,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2593100,"kb_used_data":2417604,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":91152052,"statfs":{"total":95995035648,"available":93339701248,"internally_reserved":0,"allocated":2475626496,"data_stored":2474557478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179698119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724876,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613112,"kb_used_data":361328,"kb_used_omap":1,"kb_used_meta":251774,"kb_avail":93132040,"statfs":{"total":95995035648,"available":95367208960,"internally_reserved":0,"allocated":369999872,"data_stored":368742086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":257817030},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986512,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836356,"kb_used_data":466428,"kb_used_omap":1,"kb_used_meta":369918,"kb_avail":92908796,"statfs":{"total":95995035648,"available":95138607104,"internally_reserved":0,"allocated":477622272,"data_stored":476356437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":378796486},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435404,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523572,"kb_used_data":1112876,"kb_used_omap":5,"kb_used_meta":410682,"kb_avail":92221580,"statfs":{"total":95995035648,"available":94434897920,"internally_reserved":0,"allocated":1139585024,"data_stored":1138330843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":420538441},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598930,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1125300,"kb_used_data":652140,"kb_used_omap":7,"kb_used_meta":473144,"kb_avail":92619852,"statfs":{"total":95995035648,"available":94842728448,"internally_reserved":0,"allocated":667791360,"data_stored":666537635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":484499629},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762457,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101036,"kb_used_data":754212,"kb_used_omap":7,"kb_used_meta":346808,"kb_avail":92644116,"statfs":{"total":95995035648,"available":94867574784,"internally_reserved":0,"allocated":772313088,"data_stored":771066685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":355131809},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925983,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1350400,"kb_used_data":921656,"kb_used_omap":7,"kb_used_meta":428728,"kb_avail":92394752,"statfs":{"total":95995035648,"available":94612226048,"internally_reserved":0,"allocated":943775744,"data_stored":942510555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":439018076},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":938082304,"data_stored":938082304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":743440384,"data_stored":743440384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":675020800,"data_stored":675020800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1075183616,"data_stored":1075183616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":495648768,"data_stored":495648768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":376832000,"data_stored":376832000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":478478336,"data_stored":478478336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:30:33.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T17:30:33.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:33.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:33.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:34.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:34.060 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:33] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-08-17T17:30:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:34.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.296 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:34.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:34 smithi064 ceph-mon[104516]: pgmap v721: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 12175/114705 objects degraded (10.614%); 7/114705 objects misplaced (0.006%) 2024-08-17T17:30:34.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:34 smithi064 ceph-mon[109246]: pgmap v721: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 12175/114705 objects degraded (10.614%); 7/114705 objects misplaced (0.006%) 2024-08-17T17:30:34.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:34.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:34.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:34.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:34 smithi175 ceph-mon[101434]: pgmap v721: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 12175/114705 objects degraded (10.614%); 7/114705 objects misplaced (0.006%) 2024-08-17T17:30:34.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:34.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 46334 46318 120.598 93.5625 0.0160968 0.00828602 2024-08-17T17:30:34.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:34.912 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:34.925 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:35.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:35.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:35.196 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:35.298 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.323 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:35.448 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:35.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:35.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:35.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 47867 47851 119.606 95.8125 0.0114675 0.0083554 2024-08-17T17:30:35.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:35.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:35.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:35.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:35.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:36.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:36.116 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:36.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:36.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:36.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:36.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:36.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:36.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:36.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:36 smithi064 ceph-mon[104516]: pgmap v722: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14043/133026 objects degraded (10.557%); 7/133026 objects misplaced (0.005%) 2024-08-17T17:30:36.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:36 smithi064 ceph-mon[109246]: pgmap v722: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14043/133026 objects degraded (10.557%); 7/133026 objects misplaced (0.005%) 2024-08-17T17:30:36.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:36 smithi175 ceph-mon[101434]: pgmap v722: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14043/133026 objects degraded (10.557%); 7/133026 objects misplaced (0.005%) 2024-08-17T17:30:36.692 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 49328 49312 118.518 91.3125 0.00719863 0.00843092 2024-08-17T17:30:36.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:36.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:36.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:36.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:37.038 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:37.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:37.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:37.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:37.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:37.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:37.394 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:37.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:37.543 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:37.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:37.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:37.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:37.780 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 50760 50744 117.443 89.5 0.00650981 0.00850875 2024-08-17T17:30:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:37.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:37.920 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:37.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:38.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.082 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:38.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:38.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:38.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:38.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:38.352 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:38.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: pgmap v723: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 14528/137253 objects degraded (10.585%); 7/137253 objects misplaced (0.005%) 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1252099852' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 14528/137253 objects degraded (10.585%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: pgmap v723: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 14528/137253 objects degraded (10.585%); 7/137253 objects misplaced (0.005%) 2024-08-17T17:30:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1252099852' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 14528/137253 objects degraded (10.585%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:38.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:38.604 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:38.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: pgmap v723: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 14528/137253 objects degraded (10.585%); 7/137253 objects misplaced (0.005%) 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1252099852' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 14528/137253 objects degraded (10.585%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:38.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 52195 52179 116.451 89.6875 0.0156431 0.00858106 2024-08-17T17:30:38.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:38.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:38.836 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:30:38.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:30:38.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:30:38.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:38.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:38.839 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:38.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:38.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:38.909 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.010 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.138 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:39.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:39.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:39.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 53607 53591 115.479 88.25 0.00800297 0.00865366 2024-08-17T17:30:39.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:39.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:39.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:39.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:40.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.096 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:40.173 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:40.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:40.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:40.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:40.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:40 smithi064 ceph-mon[104516]: pgmap v724: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14937/146976 objects degraded (10.163%); 7/146976 objects misplaced (0.005%) 2024-08-17T17:30:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:40 smithi064 ceph-mon[109246]: pgmap v724: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14937/146976 objects degraded (10.163%); 7/146976 objects misplaced (0.005%) 2024-08-17T17:30:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:40.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:40.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:40.619 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:40.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:40 smithi175 ceph-mon[101434]: pgmap v724: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 14937/146976 objects degraded (10.163%); 7/146976 objects misplaced (0.005%) 2024-08-17T17:30:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:40.744 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.768 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 55060 55044 114.656 90.8125 0.00599746 0.00871587 2024-08-17T17:30:40.870 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:41.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:41.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:41.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:41.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:41.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:41.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:41.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:41.539 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:41.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:41.678 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:41.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 56380 56364 113.619 82.5 0.00377023 0.00879547 2024-08-17T17:30:41.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:41.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.806 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:30:41.806 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:30:41.806 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:30:41.806 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:30:41.807 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:30:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:41.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:41.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:41.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:42.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:42.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:42.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:42.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:42.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:42.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:42.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:42.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:42.394 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:30:42.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:42.478 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:42.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:42 smithi064 ceph-mon[104516]: pgmap v725: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 16319/156729 objects degraded (10.412%); 7/156729 objects misplaced (0.004%) 2024-08-17T17:30:42.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:42.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:42 smithi064 ceph-mon[109246]: pgmap v725: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 16319/156729 objects degraded (10.412%); 7/156729 objects misplaced (0.004%) 2024-08-17T17:30:42.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:42.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:42.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:30:42.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:42 smithi175 ceph-mon[101434]: pgmap v725: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 16319/156729 objects degraded (10.412%); 7/156729 objects misplaced (0.004%) 2024-08-17T17:30:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:42.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:42.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:42.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 57745 57729 112.734 85.3125 0.0137476 0.00886534 2024-08-17T17:30:42.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:42.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:42.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:43.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.135 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:43.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:43.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:43.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:43 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:43 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:43 smithi175 ceph-mon[101434]: osdmap e235: 8 total, 7 up, 8 in 2024-08-17T17:30:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 16319/156729 objects degraded (10.412%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[104516]: osdmap e235: 8 total, 7 up, 8 in 2024-08-17T17:30:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 16319/156729 objects degraded (10.412%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2349419796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[109246]: osdmap e235: 8 total, 7 up, 8 in 2024-08-17T17:30:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 16319/156729 objects degraded (10.412%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:43.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:43.750 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:43.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 59104 59088 111.891 84.9375 0.00776617 0.00893206 2024-08-17T17:30:43.852 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:43.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:43.892 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:43.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:44.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:44.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:43] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-08-17T17:30:44.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:44.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:44.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:44.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:44.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:44.548 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.571 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:44.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:44 smithi175 ceph-mon[101434]: pgmap v727: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16750/159456 objects degraded (10.504%); 7/159456 objects misplaced (0.004%) 2024-08-17T17:30:44.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:44 smithi064 ceph-mon[104516]: pgmap v727: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16750/159456 objects degraded (10.504%); 7/159456 objects misplaced (0.004%) 2024-08-17T17:30:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:44 smithi064 ceph-mon[109246]: pgmap v727: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16750/159456 objects degraded (10.504%); 7/159456 objects misplaced (0.004%) 2024-08-17T17:30:44.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.716 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:44.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:44.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 60410 60394 111.001 81.625 0.0196257 0.0090036 2024-08-17T17:30:44.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:44.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:45.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:45.097 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:45.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:45.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:45.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:45.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:45.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:30:45.470 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:45.490 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:45.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:45.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:45.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:45.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:45.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 61593 61577 109.942 73.9375 0.00698076 0.00908932 2024-08-17T17:30:45.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:45.918 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:46.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:46.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:46.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:46.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:46.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:46.296 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:46.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:46.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:46.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:46.539 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:46.562 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:46 smithi175 ceph-mon[101434]: pgmap v728: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 18484/176046 objects degraded (10.500%); 7/176046 objects misplaced (0.004%) 2024-08-17T17:30:46.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:46 smithi064 ceph-mon[104516]: pgmap v728: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 18484/176046 objects degraded (10.500%); 7/176046 objects misplaced (0.004%) 2024-08-17T17:30:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:46 smithi064 ceph-mon[109246]: pgmap v728: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 18484/176046 objects degraded (10.500%); 7/176046 objects misplaced (0.004%) 2024-08-17T17:30:46.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:46.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:46.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 62889 62873 109.138 81 0.0173389 0.00915814 2024-08-17T17:30:46.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:46.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:46.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:46.974 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:47.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:47.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:47.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:47.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:47.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:47.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.372 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:47.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:47.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:47.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:47.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:47.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:47.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:47.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 64002 63986 108.068 69.5625 0.00751571 0.0092484 2024-08-17T17:30:47.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:47.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:47.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:48.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:48.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:48.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:48.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:48.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:48.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:48.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:48.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:48.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:48.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:48.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:48 smithi064 ceph-mon[104516]: pgmap v729: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 18909/179769 objects degraded (10.518%); 7/179769 objects misplaced (0.004%) 2024-08-17T17:30:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 18909/179769 objects degraded (10.518%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:48 smithi064 ceph-mon[109246]: pgmap v729: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 18909/179769 objects degraded (10.518%); 7/179769 objects misplaced (0.004%) 2024-08-17T17:30:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 18909/179769 objects degraded (10.518%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:48.711 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:48.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 65256 65240 107.286 78.375 0.0124309 0.00931539 2024-08-17T17:30:48.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:48.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:48.907 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:48 smithi175 ceph-mon[101434]: pgmap v729: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 18909/179769 objects degraded (10.518%); 7/179769 objects misplaced (0.004%) 2024-08-17T17:30:48.908 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 18909/179769 objects degraded (10.518%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:48.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:48.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:49.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.135 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:30:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=cleanup t=2024-08-17T17:30:48.905980464Z level=info msg="Completed cleanup jobs" duration=18.749152ms 2024-08-17T17:30:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:30:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana.update.checker t=2024-08-17T17:30:49.041121373Z level=info msg="Update check succeeded" duration=75.869148ms 2024-08-17T17:30:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:30:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugins.update.checker t=2024-08-17T17:30:49.059112585Z level=info msg="Update check succeeded" duration=67.635218ms 2024-08-17T17:30:49.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:49.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:49.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:49.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:49.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:49.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.780 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 66325 66309 106.249 66.8125 0.00567595 0.00940554 2024-08-17T17:30:49.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:49.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:49.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:50.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:50.099 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:50.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:50.225 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:50.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:50.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:50.507 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:50.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:50.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:50 smithi064 ceph-mon[104516]: pgmap v730: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19267/187680 objects degraded (10.266%); 7/187680 objects misplaced (0.004%) 2024-08-17T17:30:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:50 smithi064 ceph-mon[109246]: pgmap v730: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19267/187680 objects degraded (10.266%); 7/187680 objects misplaced (0.004%) 2024-08-17T17:30:50.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:50.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:30:50.778985+0000 min lat: 0.00120552 max lat: 0.359763 avg lat: 0.00947785 2024-08-17T17:30:50.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:30:50.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 67512 67496 105.447 74.1875 0.0152996 0.00947785 2024-08-17T17:30:50.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:50.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:50.908 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:50.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:50 smithi175 ceph-mon[101434]: pgmap v730: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 19267/187680 objects degraded (10.266%); 7/187680 objects misplaced (0.004%) 2024-08-17T17:30:51.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:51.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:51.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.434 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:51.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:51.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:51.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 68559 68543 104.471 65.4375 0.00749486 0.00956708 2024-08-17T17:30:51.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.817 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:51.826 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.928 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:51.951 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:52.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:52.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:52.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:52.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:52.231 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:52.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:52.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:52.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:52.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:52 smithi064 ceph-mon[104516]: pgmap v731: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 20412/195837 objects degraded (10.423%); 7/195837 objects misplaced (0.004%) 2024-08-17T17:30:52.584 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:52 smithi064 ceph-mon[109246]: pgmap v731: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 20412/195837 objects degraded (10.423%); 7/195837 objects misplaced (0.004%) 2024-08-17T17:30:52.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:52.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:52.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:52.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:52.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 69688 69672 103.663 70.5625 0.0109294 0.00964147 2024-08-17T17:30:52.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:52.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:52 smithi175 ceph-mon[101434]: pgmap v731: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 20412/195837 objects degraded (10.423%); 7/195837 objects misplaced (0.004%) 2024-08-17T17:30:52.990 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:53.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:53.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:53.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:53.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:53.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:53.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:53.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:53.432 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pgp_num to 1 2024-08-17T17:30:53.518 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:53.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.683 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:53.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 20412/195837 objects degraded (10.423%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 20412/195837 objects degraded (10.423%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:53.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 70947 70931 103.082 78.6875 0.0145818 0.00969613 2024-08-17T17:30:53.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:53.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:53.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 20412/195837 objects degraded (10.423%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:53.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:54.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:54.109 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:30:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:30:53] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-08-17T17:30:54.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:54.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:54.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.404 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:54.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:54.530 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:54.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:54.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[104516]: pgmap v732: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20783/198078 objects degraded (10.492%); 7/198078 objects misplaced (0.004%) 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[104516]: osdmap e236: 8 total, 7 up, 8 in 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[109246]: pgmap v732: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20783/198078 objects degraded (10.492%); 7/198078 objects misplaced (0.004%) 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:54 smithi064 ceph-mon[109246]: osdmap e236: 8 total, 7 up, 8 in 2024-08-17T17:30:54.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 72137 72121 102.429 74.375 0.00936632 0.00975674 2024-08-17T17:30:54.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:30:54.790 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:54.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:30:54.827 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:54.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:30:54.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:54 smithi175 ceph-mon[101434]: pgmap v732: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20783/198078 objects degraded (10.492%); 7/198078 objects misplaced (0.004%) 2024-08-17T17:30:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/272708863' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:54 smithi175 ceph-mon[101434]: osdmap e236: 8 total, 7 up, 8 in 2024-08-17T17:30:54.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:30:54.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:54.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:54.945 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:30:54.945 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:54.960 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:54.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:30:54.975 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:54.975 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:55.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:30:55.018 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:55.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:55.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:55.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:55.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:55.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:55.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:55.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:55.523 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:55.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/136611006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/136611006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:55.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:55.733 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:55.733 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:55.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:55.780 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 73256 73240 101.707 69.9375 0.0128714 0.00982662 2024-08-17T17:30:55.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/136611006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:55.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:55.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:56.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:56.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:56.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:56.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:56.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:56.349 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:56.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:56.489 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:56.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:56.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:56.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:56.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:56 smithi064 ceph-mon[104516]: pgmap v734: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 22315/211773 objects degraded (10.537%); 7/211773 objects misplaced (0.003%) 2024-08-17T17:30:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:56 smithi064 ceph-mon[109246]: pgmap v734: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 22315/211773 objects degraded (10.537%); 7/211773 objects misplaced (0.003%) 2024-08-17T17:30:56.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:56.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:56.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 74483 74467 101.163 76.6875 0.0119749 0.00988017 2024-08-17T17:30:56.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:56 smithi175 ceph-mon[101434]: pgmap v734: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 22315/211773 objects degraded (10.537%); 7/211773 objects misplaced (0.003%) 2024-08-17T17:30:56.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:56.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:57.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:57.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:57.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:57.166 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:57.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:57.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:57.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:30:57.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:57.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:57.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:57.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:57.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:57.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 75663 75647 100.573 73.75 0.00959012 0.00993701 2024-08-17T17:30:57.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:57.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:58.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:58.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:58.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:58.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:58.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:58.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:58.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:58.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:58.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:58.683 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:58.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:58.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 76825 76809 99.991 72.625 0.00424659 0.00999505 2024-08-17T17:30:58.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:58.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:58 smithi175 ceph-mon[101434]: pgmap v735: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 22687/215052 objects degraded (10.550%); 7/215052 objects misplaced (0.003%) 2024-08-17T17:30:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:30:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 22687/215052 objects degraded (10.550%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:58.920 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:58 smithi064 ceph-mon[104516]: pgmap v735: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 22687/215052 objects degraded (10.550%); 7/215052 objects misplaced (0.003%) 2024-08-17T17:30:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:30:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 22687/215052 objects degraded (10.550%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:58 smithi064 ceph-mon[109246]: pgmap v735: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 22687/215052 objects degraded (10.550%); 7/215052 objects misplaced (0.003%) 2024-08-17T17:30:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:30:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 22687/215052 objects degraded (10.550%), 7 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:58.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:59.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:59.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:59.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:59.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:59.375 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:59.375 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:59.416 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:59.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:59.517 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:59.517 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:59.518 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:59.518 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T17:30:59.518 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:30:59.525 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:59.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:59.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:59.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:59.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 77985 77969 99.4299 72.5 0.00926977 0.0100524 2024-08-17T17:30:59.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:30:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:59.885 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:30:59 smithi175 systemd[1]: Stopping Ceph osd.5 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:30:59.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:59.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:30:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:00.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:00.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:00.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:00.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:30:59 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[45407]: 2024-08-17T17:30:59.882+0000 7f2ab9a3a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:31:00.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:30:59 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[45407]: 2024-08-17T17:30:59.882+0000 7f2ab9a3a640 -1 osd.5 236 *** Got signal Terminated *** 2024-08-17T17:31:00.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:30:59 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[45407]: 2024-08-17T17:30:59.882+0000 7f2ab9a3a640 -1 osd.5 236 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:31:00.174 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:00.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:00.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:00.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:00.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:00.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:00.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:00.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:00.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:00.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:00.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 78846 78830 98.5174 53.8125 0.00321877 0.0100933 2024-08-17T17:31:00.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:00 smithi064 ceph-mon[104516]: pgmap v736: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23059/222906 objects degraded (10.345%); 7/222906 objects misplaced (0.003%) 2024-08-17T17:31:00.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:00 smithi064 ceph-mon[104516]: osd.5 marked itself down and dead 2024-08-17T17:31:00.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:00 smithi064 ceph-mon[109246]: pgmap v736: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23059/222906 objects degraded (10.345%); 7/222906 objects misplaced (0.003%) 2024-08-17T17:31:00.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:00 smithi064 ceph-mon[109246]: osd.5 marked itself down and dead 2024-08-17T17:31:00.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:00.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:00 smithi175 ceph-mon[101434]: pgmap v736: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 23059/222906 objects degraded (10.345%); 7/222906 objects misplaced (0.003%) 2024-08-17T17:31:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:00 smithi175 ceph-mon[101434]: osd.5 marked itself down and dead 2024-08-17T17:31:00.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:00.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:01.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:01.105 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:01.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:01.228 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:01.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:01.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:01.352 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:01 smithi175 podman[158030]: 2024-08-17 17:31:01.063543493 +0000 UTC m=+1.308999428 container died 656b5f8b3cd0a418ea50541a080f5fc6081a71e483392fcbe181777c428fd5d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-ced649f, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:31:01.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:01.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:01.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:01.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:01.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:01.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:01 smithi064 ceph-mon[104516]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:31:01.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:01 smithi064 ceph-mon[104516]: osdmap e237: 8 total, 6 up, 8 in 2024-08-17T17:31:01.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:01 smithi064 ceph-mon[109246]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:31:01.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:01 smithi064 ceph-mon[109246]: osdmap e237: 8 total, 6 up, 8 in 2024-08-17T17:31:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:01 smithi175 ceph-mon[101434]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:31:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:01 smithi175 ceph-mon[101434]: osdmap e237: 8 total, 6 up, 8 in 2024-08-17T17:31:01.661 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:01 smithi175 podman[158030]: 2024-08-17 17:31:01.51699078 +0000 UTC m=+1.762446716 container cleanup 656b5f8b3cd0a418ea50541a080f5fc6081a71e483392fcbe181777c428fd5d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:31:01.661 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:01 smithi175 bash[158030]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5 2024-08-17T17:31:01.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:01.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:01.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 78967 78951 96.7342 7.5625 0.00435079 0.010257 2024-08-17T17:31:01.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:01.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:01.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:01.993 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.116 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.149 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:01 smithi175 podman[158115]: 2024-08-17 17:31:01.802207361 +0000 UTC m=+0.734557077 container remove 656b5f8b3cd0a418ea50541a080f5fc6081a71e483392fcbe181777c428fd5d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-17T17:31:02.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:02.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:02.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:02.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:02.400 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.051403772 +0000 UTC m=+0.017665289 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:31:02.401 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.178100371 +0000 UTC m=+0.144361887 container create eda54198dfdac27ce797c5da6ca78a60504992ccca45446e45907d1e2e4f7bbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T17:31:02.401 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.397260589 +0000 UTC m=+0.363522112 container init eda54198dfdac27ce797c5da6ca78a60504992ccca45446e45907d1e2e4f7bbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f) 2024-08-17T17:31:02.401 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.40036253 +0000 UTC m=+0.366624053 container start eda54198dfdac27ce797c5da6ca78a60504992ccca45446e45907d1e2e4f7bbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-08-17T17:31:02.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:02.489 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:02.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:02 smithi175 ceph-mon[101434]: pgmap v738: 89 pgs: 6 stale+active+clean, 5 stale+active+clean+remapped, 4 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 24121/229323 objects degraded (10.518%); 7/229323 objects misplaced (0.003%) 2024-08-17T17:31:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:02 smithi175 ceph-mon[101434]: osdmap e238: 8 total, 6 up, 8 in 2024-08-17T17:31:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1440252504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:02 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.662 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.497339074 +0000 UTC m=+0.463600604 container attach eda54198dfdac27ce797c5da6ca78a60504992ccca45446e45907d1e2e4f7bbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:31:02.662 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:02 smithi175 podman[158236]: 2024-08-17 17:31:02.58526181 +0000 UTC m=+0.551523332 container died eda54198dfdac27ce797c5da6ca78a60504992ccca45446e45907d1e2e4f7bbf (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:31:02.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:02.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:02.746 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:02.746 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[104516]: pgmap v738: 89 pgs: 6 stale+active+clean, 5 stale+active+clean+remapped, 4 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 24121/229323 objects degraded (10.518%); 7/229323 objects misplaced (0.003%) 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[104516]: osdmap e238: 8 total, 6 up, 8 in 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1440252504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[109246]: pgmap v738: 89 pgs: 6 stale+active+clean, 5 stale+active+clean+remapped, 4 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 24121/229323 objects degraded (10.518%); 7/229323 objects misplaced (0.003%) 2024-08-17T17:31:02.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[109246]: osdmap e238: 8 total, 6 up, 8 in 2024-08-17T17:31:02.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1440252504' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:02 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:02.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 78967 78951 94.874 0 - 0.010257 2024-08-17T17:31:02.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:03.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:03.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T17:31:03.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:03.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:03.241 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:31:03.241 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T17:31:03.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:03.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:03.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:03.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:03.587 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:03.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:03.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.738 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 78967 78951 93.0841 0 - 0.010257 2024-08-17T17:31:03.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:03.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:03 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:03 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:03 smithi175 ceph-mon[101434]: osdmap e239: 8 total, 6 up, 8 in 2024-08-17T17:31:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 24121/229323 objects degraded (10.518%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[104516]: osdmap e239: 8 total, 6 up, 8 in 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 24121/229323 objects degraded (10.518%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[109246]: osdmap e239: 8 total, 6 up, 8 in 2024-08-17T17:31:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 24121/229323 objects degraded (10.518%), 7 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:03.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:03.993 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:04.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:04.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:04.224 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:04.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:04.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:04.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:03] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-08-17T17:31:04.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:04.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:04.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:04.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:04 smithi175 ceph-mon[101434]: pgmap v741: 89 pgs: 2 active+undersized+remapped, 2 peering, 1 remapped+peering, 3 stale+active+clean, 4 stale+active+clean+remapped, 2 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 24411/231186 objects degraded (10.559%); 7/231186 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2024-08-17T17:31:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:04 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:31:04.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 78967 78951 91.3604 0 - 0.010257 2024-08-17T17:31:04.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:04 smithi064 ceph-mon[104516]: pgmap v741: 89 pgs: 2 active+undersized+remapped, 2 peering, 1 remapped+peering, 3 stale+active+clean, 4 stale+active+clean+remapped, 2 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 24411/231186 objects degraded (10.559%); 7/231186 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2024-08-17T17:31:04.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:04 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:31:04.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:04 smithi064 ceph-mon[109246]: pgmap v741: 89 pgs: 2 active+undersized+remapped, 2 peering, 1 remapped+peering, 3 stale+active+clean, 4 stale+active+clean+remapped, 2 active+clean+remapped, 16 active+undersized, 8 active+undersized+degraded, 51 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 24411/231186 objects degraded (10.559%); 7/231186 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2024-08-17T17:31:04.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:04 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:31:04.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:04.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:04.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:05.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:05.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:05.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:05.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:05.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:05.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:31:05.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:05.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:05.395 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:31:05.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.514 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:05.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:05.740 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 78967 78951 89.6995 0 - 0.010257 2024-08-17T17:31:05.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:06.019 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:06.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:06.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:06.174 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:06.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:06.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:06.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:06.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:06.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:06.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:06.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:06.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:06.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.776 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:06.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 78967 78951 88.0979 0 - 0.010257 2024-08-17T17:31:06.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:06 smithi064 ceph-mon[104516]: pgmap v742: 89 pgs: 7 active+undersized+remapped, 2 peering, 1 remapped+peering, 1 stale+active+clean+remapped, 22 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 44627/236910 objects degraded (18.837%); 12/236910 objects misplaced (0.005%); 11 KiB/s, 0 objects/s recovering 2024-08-17T17:31:06.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:06 smithi064 ceph-mon[109246]: pgmap v742: 89 pgs: 7 active+undersized+remapped, 2 peering, 1 remapped+peering, 1 stale+active+clean+remapped, 22 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 44627/236910 objects degraded (18.837%); 12/236910 objects misplaced (0.005%); 11 KiB/s, 0 objects/s recovering 2024-08-17T17:31:06.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:06.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:06 smithi175 ceph-mon[101434]: pgmap v742: 89 pgs: 7 active+undersized+remapped, 2 peering, 1 remapped+peering, 1 stale+active+clean+remapped, 22 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 44627/236910 objects degraded (18.837%); 12/236910 objects misplaced (0.005%); 11 KiB/s, 0 objects/s recovering 2024-08-17T17:31:07.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:07.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:07.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:07.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:07.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:07.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:07.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:07.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:07.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:07.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:07.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:07.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:07 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:31:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:07 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:31:07.732 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:07.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:07.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:07.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 78967 78951 86.5525 0 - 0.010257 2024-08-17T17:31:07.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:07.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:07 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:31:08.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:08.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:31:08.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:08.242 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6, 5] live_osds: [0, 1, 2, 3, 4, 7] 2024-08-17T17:31:08.242 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:31:08.242 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-08-17T17:31:08.242 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-08-17T17:31:08.242 DEBUG:teuthology.orchestra.run.smithi175:osd.5> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T17:31:08.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl start ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:31:08.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:08.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:08.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.547 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 systemd[1]: Starting Ceph osd.5 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:31:08.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:08.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:08.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:08.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:08.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 78967 78951 85.0603 0 - 0.010257 2024-08-17T17:31:08.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:08.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:08.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:08 smithi064 ceph-mon[104516]: pgmap v743: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 389 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:31:08.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 44630/236937 objects degraded (18.836%), 13 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:08.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:08 smithi064 ceph-mon[109246]: pgmap v743: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 389 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:31:08.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 44630/236937 objects degraded (18.836%), 13 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:08.856 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:08 smithi175 ceph-mon[101434]: pgmap v743: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 24 MiB/s wr, 389 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:31:08.856 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 44630/236937 objects degraded (18.836%), 13 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:08.856 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 podman[158680]: 2024-08-17 17:31:08.463381887 +0000 UTC m=+0.017600884 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:31:08.857 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 podman[158680]: 2024-08-17 17:31:08.57144451 +0000 UTC m=+0.125663509 container create 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:31:08.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:08.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:09.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:09.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:09.134 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:09.135 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:31:09.152 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 podman[158680]: 2024-08-17 17:31:08.855386157 +0000 UTC m=+0.409605146 container init 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:31:09.152 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 podman[158680]: 2024-08-17 17:31:08.859832592 +0000 UTC m=+0.414051591 container start 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:31:09.152 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:08 smithi175 podman[158680]: 2024-08-17 17:31:08.930269801 +0000 UTC m=+0.484488800 container attach 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:31:09.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:09.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:09.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:09.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:09.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:09.343 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-17T17:31:09.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate[158695]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 bash[158680]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-17T17:31:09.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 podman[158680]: 2024-08-17 17:31:09.202434972 +0000 UTC m=+0.756653964 container died 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_BRANCH=HEAD) 2024-08-17T17:31:09.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:09.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:09.548 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:09.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:09.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:09.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:09.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3379051935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:09.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3379051935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:09.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 78967 78951 83.6187 0 - 0.010257 2024-08-17T17:31:09.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:09.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3379051935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:09.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:09.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.067 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:10.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:10.287 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:09 smithi175 podman[158858]: 2024-08-17 17:31:09.927657042 +0000 UTC m=+0.719591879 container remove 6ac8640393b93ad1c79ae902de30548a5b67adfbf196d1e6b669b9a88753bd40 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T17:31:10.287 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 podman[158953]: 2024-08-17 17:31:10.046726484 +0000 UTC m=+0.017254702 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:31:10.287 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 podman[158953]: 2024-08-17 17:31:10.160979322 +0000 UTC m=+0.131507531 container create bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-17T17:31:10.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.314 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:10.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:10.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:10.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:10.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:31:10.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:10.757 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:10.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:31:10.784390+0000 min lat: 0.00120552 max lat: 1.04459 avg lat: 0.010257 2024-08-17T17:31:10.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:10.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 78967 78951 82.2251 0 - 0.010257 2024-08-17T17:31:10.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.879 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:10 smithi175 ceph-mon[101434]: pgmap v744: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 317 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:31:10.879 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:10.879 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:10.880 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 podman[158953]: 2024-08-17 17:31:10.41355224 +0000 UTC m=+0.384080467 container init bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T17:31:10.880 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 podman[158953]: 2024-08-17 17:31:10.416739659 +0000 UTC m=+0.387267873 container start bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-17T17:31:10.880 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 ceph-osd[158989]: -- 172.21.15.175:0/3277242696 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556a7d03f1e0 con 0x556a7ddbe400 2024-08-17T17:31:10.880 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 bash[158953]: bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac 2024-08-17T17:31:10.880 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 systemd[1]: Started Ceph osd.5 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:31:10.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[104516]: pgmap v744: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 317 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:31:10.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:10.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:10.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[109246]: pgmap v744: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 317 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:31:10.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:10.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:10.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:10.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:11.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:11.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:11.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:11.116 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:31:11.116 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:31:11.116 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:31:11.116 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:31:11.116 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:31:11.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:10 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:31:10.952+0000 7f925df48640 -1 Falling back to public interface 2024-08-17T17:31:11.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:11.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:11.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:11.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:11.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:11.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:11.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:11.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:11.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:11.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:11.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 78967 78951 80.8772 0 - 0.010257 2024-08-17T17:31:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:11.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:31:11.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:12.053 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:12.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:12.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:12.301 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:12.301 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.302 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:12.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:12.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:12.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:12.507 INFO:teuthology.orchestra.run.smithi175.stderr:no valid command found; 10 closest matches: 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:0 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:1 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:2 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:abort 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:assert 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:config diff 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:config diff get 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:config get 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:config help [] 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:config set ... 2024-08-17T17:31:12.508 INFO:teuthology.orchestra.run.smithi175.stderr:admin_socket: invalid command 2024-08-17T17:31:12.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:12.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:12.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:12.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:12.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 78967 78951 79.5729 0 - 0.010257 2024-08-17T17:31:12.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.884 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:12 smithi175 ceph-mon[101434]: pgmap v745: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.7 KiB/s, 0 objects/s recovering 2024-08-17T17:31:12.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.920 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:12.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:12 smithi064 ceph-mon[104516]: pgmap v745: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.7 KiB/s, 0 objects/s recovering 2024-08-17T17:31:12.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:12 smithi064 ceph-mon[109246]: pgmap v745: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.7 KiB/s, 0 objects/s recovering 2024-08-17T17:31:13.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:13.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:31:13.110 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-08-17T17:31:13.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.169 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:13.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:13.291 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:13.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:13.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:13.436 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:13.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:13.539 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:13.560 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:31:13.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:13.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:13.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 78967 78951 78.3099 0 - 0.010257 2024-08-17T17:31:13.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:13.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:13.910 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:13.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:13.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:13.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:14.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:14.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:14.135 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:14.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:14.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:14.282 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:14.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:31:14.346 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:13] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:31:14.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:14.405 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:14.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:14.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:14.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[104516]: pgmap v746: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 MiB/s wr, 182 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.1 KiB/s, 0 objects/s recovering 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[104516]: osdmap e240: 8 total, 6 up, 8 in 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[109246]: pgmap v746: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 MiB/s wr, 182 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.1 KiB/s, 0 objects/s recovering 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:14.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:14 smithi064 ceph-mon[109246]: osdmap e240: 8 total, 6 up, 8 in 2024-08-17T17:31:14.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:14 smithi175 ceph-mon[101434]: pgmap v746: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11 MiB/s wr, 182 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%); 6.1 KiB/s, 0 objects/s recovering 2024-08-17T17:31:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:14 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1213242153' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:14 smithi175 ceph-mon[101434]: osdmap e240: 8 total, 6 up, 8 in 2024-08-17T17:31:14.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:14.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:14.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 78967 78951 77.0864 0 - 0.010257 2024-08-17T17:31:14.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:14.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:15.025 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:15.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:15.148 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:15.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:15.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:15.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:15.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:15.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:15.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:15.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:15.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 78967 78951 75.9006 0 - 0.010257 2024-08-17T17:31:15.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:15 smithi064 ceph-mon[109246]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:15.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:15 smithi064 ceph-mon[109246]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:15.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:15 smithi064 ceph-mon[104516]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:15.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:15 smithi064 ceph-mon[104516]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:15.860 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:15.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:15.886 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:31:15.886 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:31:15.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:31:15.886 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:31:15.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131921 5.a 5:5beda773:::benchmark_data_smithi064_147470_object131920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 553.84194080400005, 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4918728809999999, 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131921 2024-08-17T17:31:15.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042044+0000", 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042045+0000", 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9999999999999997e-07 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042067+0000", 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1823000000000001e-05 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042076+0000", 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9230000000000008e-06 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.042082+0000", 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8850000000000004e-06 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.045641+0000", 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035593339999999999 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.049865+0000", 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042238900000000001 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.050039+0000", 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017439300000000001 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.065709+0000", 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015669374999999999 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.065715+0000", 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5409999999999997e-06 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040390+0000", 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97467535299999997 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040408+0000", 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7765e-05 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.507316+0000", 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.466907371 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.509272+0000", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019564259999999998 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533838+0000", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024566119000000001 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533917+0000", 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9083000000000002e-05 2024-08-17T17:31:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131926 5.a 5:50f30e39:::benchmark_data_smithi064_147470_object131925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 553.82280385700005, 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4772495339999998, 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.895 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131926 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061181+0000", 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061183+0000", 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6920000000000001e-06 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061208+0000", 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4828999999999999e-05 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061218+0000", 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560000000000001e-05 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.061225+0000", 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0659999999999998e-06 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.066388+0000", 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005162725 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.066397+0000", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1779999999999992e-06 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040484+0000", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97408712900000005 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.040495+0000", 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0538e-05 2024-08-17T17:31:15.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515768+0000", 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.47527309 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515914+0000", 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014613699999999999 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.516046+0000", 2024-08-17T17:31:15.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013153899999999999 2024-08-17T17:31:15.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.533966+0000", 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017920177999999998 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534025+0000", 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8739000000000001e-05 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534037+0000", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1953000000000001e-05 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.534069+0000", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1854999999999999e-05 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.538431+0000", 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043623259999999997 2024-08-17T17:31:15.903 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140758 5.a 5:5403018b:::benchmark_data_smithi064_147470_object8814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 535.74363238299998, 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234468573, 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.904 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140758 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140353+0000", 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140354+0000", 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6130000000000001e-06 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140360+0000", 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6579999999999999e-06 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140377+0000", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7433000000000001e-05 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140386+0000", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8079999999999994e-06 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145573+0000", 2024-08-17T17:31:15.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051867600000000003 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145756+0000", 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018317 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.145928+0000", 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017162699999999999 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155183+0000", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092555829999999995 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172033+0000", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016849670000000001 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230629+0000", 2024-08-17T17:31:15.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058595582 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230636+0000", 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6550000000000004e-06 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.268659+0000", 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.038022794 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269045+0000", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000385907 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374769+0000", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10572400799999999 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374821+0000", 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2305000000000002e-05 2024-08-17T17:31:15.911 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140763 5.a 5:51aa539c:::benchmark_data_smithi064_147470_object8819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 535.74358436199998, 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234340475, 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140763 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:31:15.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140401+0000", 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8100000000000003e-07 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140404+0000", 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3369999999999998e-06 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140410+0000", 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0979999999999999e-06 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.140414+0000", 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7809999999999996e-06 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150277+0000", 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098622109999999992 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150431+0000", 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154711 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.150571+0000", 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013928700000000001 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172074+0000", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02150297 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172168+0000", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4524999999999994e-05 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230661+0000", 2024-08-17T17:31:15.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058492423000000002 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230668+0000", 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.063e-06 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269162+0000", 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0384945990000001 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269387+0000", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022499099999999999 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374683+0000", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.105295499 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374741+0000", 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8498999999999997e-05 2024-08-17T17:31:15.919 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140766 5.a 5:5069331d:::benchmark_data_smithi064_147470_object8822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 535.72893585899999, 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.219602018, 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.920 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140766 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155049+0000", 2024-08-17T17:31:15.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155050+0000", 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9399999999999995e-07 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155052+0000", 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5009999999999999e-06 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155059+0000", 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4609999999999996e-06 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.155064+0000", 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2769999999999996e-06 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.162992+0000", 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079281260000000006 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.163140+0000", 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014762 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.163297+0000", 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015770199999999999 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193553+0000", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030255414000000001 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.198428+0000", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048752889999999997 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230694+0000", 2024-08-17T17:31:15.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032266112999999999 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230701+0000", 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1359999999999999e-06 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269493+0000", 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.038791429 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269585+0000", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2304000000000003e-05 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374585+0000", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.104999962 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374651+0000", 2024-08-17T17:31:15.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6190000000000002e-05 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140776 5.a 5:596336eb:::benchmark_data_smithi064_147470_object8832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 535.71163223600001, 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.201320121, 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.928 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140776 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172353+0000", 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172355+0000", 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9470000000000002e-06 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172360+0000", 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4099999999999999e-06 2024-08-17T17:31:15.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172375+0000", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4679e-05 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172382+0000", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172445+0000", 2024-08-17T17:31:15.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2534999999999997e-05 2024-08-17T17:31:15.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:15.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:15.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172619+0000", 2024-08-17T17:31:15.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173707 2024-08-17T17:31:15.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.172772+0000", 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015342599999999999 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193595+0000", 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020822811 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.198437+0000", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00484199 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230726+0000", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032288692000000001 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230733+0000", 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2599999999999999e-06 2024-08-17T17:31:15.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269619+0000", 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.03888622 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269709+0000", 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0048000000000002e-05 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.373587+0000", 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103877517 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.373673+0000", 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6349999999999998e-05 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140781 5.a 5:5cbf5a79:::benchmark_data_smithi064_147470_object8837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-17T17:31:15.938 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 535.70079956100005, 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1908856889999999, 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140781 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183186+0000", 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183187+0000", 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.065e-06 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183191+0000", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6700000000000002e-06 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183200+0000", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2050000000000002e-06 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.183205+0000", 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1080000000000001e-06 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193625+0000", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010420674 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193810+0000", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000184721 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.193983+0000", 2024-08-17T17:31:15.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173072 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.199492+0000", 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055087130000000002 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.200443+0000", 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000951205 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230758+0000", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030315023999999999 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.230765+0000", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3130000000000001e-06 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.269855+0000", 2024-08-17T17:31:15.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.039089626 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.270151+0000", 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000296497 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374011+0000", 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103859375 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374071+0000", 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0421000000000003e-05 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71399 6.13 6:ce72d6cc:::benchmark_data_smithi064_185391_object71398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73605853800001, 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.186731456, 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.946 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71399 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9200000000000001e-07 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147969+0000", 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1732e-05 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147976+0000", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147981+0000", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150092+0000", 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021112190000000001 2024-08-17T17:31:15.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150098+0000", 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9739999999999999e-06 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314529+0000", 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1644304400000001 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314695+0000", 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016647400000000001 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314863+0000", 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016801899999999999 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318404+0000", 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035409460000000001 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321548+0000", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031436210000000001 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326324+0000", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047760069999999996 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332911+0000", 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065868460000000004 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332962+0000", 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1026000000000002e-05 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334658+0000", 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016963169999999999 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71427 6.13 6:c8d80d54:::benchmark_data_smithi064_185391_object71426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73599728600001, 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1881262239999999, 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71427 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9699999999999996e-07 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148026+0000", 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7808000000000001e-05 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148033+0000", 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5409999999999997e-06 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148038+0000", 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7400000000000001e-06 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150120+0000", 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020819060000000001 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150126+0000", 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4450000000000004e-06 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316489+0000", 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1663628049999999 2024-08-17T17:31:15.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316645+0000", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015632600000000001 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316822+0000", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017728700000000001 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318448+0000", 2024-08-17T17:31:15.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001625472 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321553+0000", 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031055340000000001 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326371+0000", 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004817867 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334787+0000", 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084156920000000007 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334912+0000", 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336114+0000", 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012023540000000001 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71428 6.13 6:cafa274c:::benchmark_data_smithi064_185391_object71427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73594095600001, 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.189277785, 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.962 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71428 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148045+0000", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6899999999999998e-07 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148082+0000", 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7800999999999999e-05 2024-08-17T17:31:15.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148089+0000", 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2349999999999996e-06 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148093+0000", 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4490000000000001e-06 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150147+0000", 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020543169999999999 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150153+0000", 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7699999999999998e-06 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318492+0000", 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1683386149999999 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318638+0000", 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145742 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318823+0000", 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018568299999999999 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321525+0000", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027019539999999999 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321558+0000", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2984000000000001e-05 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332861+0000", 2024-08-17T17:31:15.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011302553 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336256+0000", 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033948149999999998 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336311+0000", 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4963000000000002e-05 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337322+0000", 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001011435 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71439 6.13 6:cdd6e9ae:::benchmark_data_smithi064_185391_object71438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.735886222, 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.190434312, 2024-08-17T17:31:15.970 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71439 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5999999999999999e-07 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148138+0000", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8883e-05 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148145+0000", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.28e-06 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148149+0000", 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2899999999999996e-06 2024-08-17T17:31:15.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150175+0000", 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020258120000000001 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150180+0000", 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6080000000000003e-06 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.319903+0000", 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.169723114 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.320064+0000", 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000160941 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.320236+0000", 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017170900000000001 2024-08-17T17:31:15.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.324936+0000", 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046998070000000003 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334730+0000", 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097942949999999997 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334747+0000", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6861e-05 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337424+0000", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026770560000000001 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337484+0000", 2024-08-17T17:31:15.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0291999999999998e-05 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338533+0000", 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001048904 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71448 6.13 6:c94d4d39:::benchmark_data_smithi064_185391_object71447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.978 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73583072100001, 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.194234171, 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71448 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148155+0000", 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2199999999999999e-07 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148193+0000", 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7851000000000001e-05 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148199+0000", 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3389999999999998e-06 2024-08-17T17:31:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148203+0000", 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2740000000000001e-06 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150202+0000", 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019990170000000001 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150208+0000", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7919999999999998e-06 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326405+0000", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1761970420000001 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326548+0000", 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142993 2024-08-17T17:31:15.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.329864+0000", 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033157310000000001 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336134+0000", 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062696519999999997 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336181+0000", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7429000000000003e-05 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336218+0000", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7048999999999997e-05 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.340929+0000", 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047111210000000004 2024-08-17T17:31:15.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.340985+0000", 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5849e-05 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342389+0000", 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00140361 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.986 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71459 6.13 6:ca93aa2f:::benchmark_data_smithi064_185391_object71458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73577533100001, 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.211085977, 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71459 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6699999999999999e-07 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148249+0000", 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8473999999999998e-05 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148256+0000", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9110000000000004e-06 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148260+0000", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3370000000000001e-06 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150230+0000", 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019701839999999998 2024-08-17T17:31:15.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150236+0000", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8279999999999999e-06 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331258+0000", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1810224380000001 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331409+0000", 2024-08-17T17:31:15.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015012099999999999 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331573+0000", 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164191 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336140+0000", 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045669209999999998 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337372+0000", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012318190000000001 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.343959+0000", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065878200000000003 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.357255+0000", 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013295493 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.357804+0000", 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054940100000000003 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359296+0000", 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001491572 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71470 6.13 6:ce05dbf9:::benchmark_data_smithi064_185391_object71469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73571893299999, 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.216359454, 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71470 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148267+0000", 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2e-07 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148305+0000", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8278999999999997e-05 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148311+0000", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8769999999999998e-06 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148315+0000", 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1010000000000002e-06 2024-08-17T17:31:15.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150258+0000", 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001942928 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150264+0000", 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7389999999999996e-06 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:15.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338570+0000", 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.188306224 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338721+0000", 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151605 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338895+0000", 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173902 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:15.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342442+0000", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035463809999999999 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353252+0000", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010810637 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356482+0000", 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032296260000000002 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359645+0000", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031634190000000002 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359807+0000", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000161846 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364626+0000", 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00481847 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.001 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71504 6.13 6:ca612f25:::benchmark_data_smithi064_185391_object71503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.735664036, 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217744341, 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71504 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7e-07 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148362+0000", 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0194000000000002e-05 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148368+0000", 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8810000000000001e-06 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148373+0000", 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6500000000000001e-06 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150286+0000", 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001912871 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150292+0000", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7119999999999997e-06 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342474+0000", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1921820140000001 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342614+0000", 2024-08-17T17:31:16.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014014600000000001 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342779+0000", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016551000000000001 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348286+0000", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055062699999999997 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356468+0000", 2024-08-17T17:31:16.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081821329999999994 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356488+0000", 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0157e-05 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364686+0000", 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081976570000000006 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364747+0000", 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1583999999999999e-05 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366065+0000", 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013181919999999999 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71518 6.13 6:cca05011:::benchmark_data_smithi064_185391_object71517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.735605085, 2024-08-17T17:31:16.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.22094334, 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71518 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148381+0000", 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9200000000000001e-07 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148421+0000", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0896999999999998e-05 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148429+0000", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148435+0000", 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3730000000000001e-06 2024-08-17T17:31:16.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150319+0000", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018839639999999999 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150325+0000", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.343995+0000", 2024-08-17T17:31:16.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1936702260000001 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.344156+0000", 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016101599999999999 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.344345+0000", 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000189328 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348329+0000", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039839280000000003 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356494+0000", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081646300000000008 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359431+0000", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029367830000000001 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366120+0000", 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066888980000000004 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366179+0000", 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9175000000000002e-05 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369323+0000", 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031445179999999998 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.016 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71568 6.13 6:ca8b9dca:::benchmark_data_smithi064_185391_object71567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73554393500001, 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.222653634, 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.017 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71568 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148442+0000", 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0900000000000002e-07 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148480+0000", 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8139999999999997e-05 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148487+0000", 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.737e-06 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148491+0000", 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6460000000000001e-06 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150352+0000", 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018606600000000001 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150358+0000", 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4760000000000003e-06 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.345703+0000", 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1953445629999999 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.345855+0000", 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015168199999999999 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.346446+0000", 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000591768 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353296+0000", 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006849656 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356500+0000", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032040150000000002 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359502+0000", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00300198 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369428+0000", 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099256210000000008 2024-08-17T17:31:16.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369597+0000", 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016884800000000001 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371095+0000", 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014983329999999999 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71616 6.13 6:cdda852e:::benchmark_data_smithi064_185391_object71615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73548789, 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225571365, 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71616 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148498+0000", 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3300000000000003e-07 2024-08-17T17:31:16.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148583+0000", 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4808000000000005e-05 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148589+0000", 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8669999999999996e-06 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148596+0000", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.024e-06 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150384+0000", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017877570000000001 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150390+0000", 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348360+0000", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.197969335 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348504+0000", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144337 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348685+0000", 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018043000000000001 2024-08-17T17:31:16.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353338+0000", 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046530759999999999 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356506+0000", 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031686169999999999 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369382+0000", 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012875985 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371198+0000", 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018155790000000001 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371260+0000", 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2675999999999994e-05 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374069+0000", 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028081479999999999 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71674 6.13 6:c8c36f7a:::benchmark_data_smithi064_185391_object71673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.73538156500001, 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2267917589999999, 2024-08-17T17:31:16.032 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71674 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9399999999999995e-07 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148644+0000", 2024-08-17T17:31:16.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9901000000000003e-05 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148651+0000", 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0539999999999997e-06 2024-08-17T17:31:16.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.036 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:15 smithi175 ceph-mon[101434]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:16.036 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:15 smithi175 ceph-mon[101434]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:31:16.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148656+0000", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.758e-06 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150415+0000", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017591340000000001 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150421+0000", 2024-08-17T17:31:16.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1090000000000003e-06 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.351716+0000", 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2012946879999999 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.351875+0000", 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159304 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.352054+0000", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017908999999999999 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353381+0000", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013267559999999999 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356512+0000", 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003131572 2024-08-17T17:31:16.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371152+0000", 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014639353000000001 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374179+0000", 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00302769 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374334+0000", 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154938 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.375395+0000", 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010609180000000001 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71683 6.13 6:cb56e99d:::benchmark_data_smithi064_185391_object71682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:31:16.041 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 255.735323607, 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2329368380000001, 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71683 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:31:16.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1800000000000001e-07 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148703+0000", 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.074e-05 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148709+0000", 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2280000000000003e-06 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148714+0000", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3469999999999998e-06 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150524+0000", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018093339999999999 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150540+0000", 2024-08-17T17:31:16.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6679000000000001e-05 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353442+0000", 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2029013989999999 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.354929+0000", 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014875380000000001 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.355153+0000", 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022341099999999999 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359447+0000", 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042943510000000001 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359556+0000", 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109091 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374127+0000", 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014571183999999999 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.379151+0000", 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005024204 2024-08-17T17:31:16.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.379515+0000", 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000363734 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.381598+0000", 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00208318 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:16.050 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:16.050 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:31:16.050 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:15 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:31:15.543+0000 7f925df48640 -1 osd.5 236 log_to_monitors true 2024-08-17T17:31:16.050 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:31:15 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:31:15.592+0000 7f9255cf2640 -1 osd.5 236 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:31:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:16.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:16.196 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:16.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:16.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:16.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:31:16.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:16.446 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:16.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:16.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:16.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:16.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[104516]: pgmap v748: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 KiB/s wr, 0 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%) 2024-08-17T17:31:16.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[104516]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:31:16.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[104516]: osdmap e241: 8 total, 6 up, 8 in 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[104516]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[104516]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[109246]: pgmap v748: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 KiB/s wr, 0 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%) 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[109246]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[109246]: osdmap e241: 8 total, 6 up, 8 in 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[109246]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:16 smithi064 ceph-mon[109246]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:16.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 78967 78951 74.7507 0 - 0.010257 2024-08-17T17:31:16.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:16.820 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:16 smithi175 ceph-mon[101434]: pgmap v748: 89 pgs: 9 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 KiB/s wr, 0 op/s; 44630/236937 objects degraded (18.836%); 14/236937 objects misplaced (0.006%) 2024-08-17T17:31:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:16 smithi175 ceph-mon[101434]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:31:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:16 smithi175 ceph-mon[101434]: osdmap e241: 8 total, 6 up, 8 in 2024-08-17T17:31:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:16 smithi175 ceph-mon[101434]: from='osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:16 smithi175 ceph-mon[101434]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:16.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:17.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.067 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:17.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:17.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:17.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:17.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:17.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:17.417 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:17.442 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:17.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:17.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:17.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:17.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:17.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 79152 79136 73.8076 0.722656 0.0110426 0.013543 2024-08-17T17:31:17.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:17.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:17 smithi175 ceph-mon[101434]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:31:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:17 smithi175 ceph-mon[101434]: osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607] boot 2024-08-17T17:31:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:17 smithi175 ceph-mon[101434]: osdmap e242: 8 total, 7 up, 8 in 2024-08-17T17:31:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:31:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[104516]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:31:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[104516]: osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607] boot 2024-08-17T17:31:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[104516]: osdmap e242: 8 total, 7 up, 8 in 2024-08-17T17:31:17.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:31:17.938 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[109246]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:31:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[109246]: osd.5 [v2:172.21.15.175:6808/1907651607,v1:172.21.15.175:6809/1907651607] boot 2024-08-17T17:31:17.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[109246]: osdmap e242: 8 total, 7 up, 8 in 2024-08-17T17:31:17.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:31:17.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:18.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.105 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:31:18.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:31:18.249 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.358 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:18.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:18.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:18.627 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:18.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:18.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:18.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:18 smithi175 ceph-mon[101434]: pgmap v751: 89 pgs: 2 remapped+peering, 7 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44630/236937 objects degraded (18.836%); 12/236937 objects misplaced (0.005%) 2024-08-17T17:31:18.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:18 smithi175 ceph-mon[101434]: osdmap e243: 8 total, 7 up, 8 in 2024-08-17T17:31:18.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 80186 80170 73.6719 64.625 0.0151856 0.0135678 2024-08-17T17:31:18.805 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:18.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:31:18.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:18.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:18 smithi064 ceph-mon[104516]: pgmap v751: 89 pgs: 2 remapped+peering, 7 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44630/236937 objects degraded (18.836%); 12/236937 objects misplaced (0.005%) 2024-08-17T17:31:18.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:18 smithi064 ceph-mon[104516]: osdmap e243: 8 total, 7 up, 8 in 2024-08-17T17:31:18.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:18 smithi064 ceph-mon[109246]: pgmap v751: 89 pgs: 2 remapped+peering, 7 active+undersized+remapped, 24 active+undersized, 1 undersized+degraded+peered, 12 active+undersized+degraded, 43 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44630/236937 objects degraded (18.836%); 12/236937 objects misplaced (0.005%) 2024-08-17T17:31:18.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:18 smithi064 ceph-mon[109246]: osdmap e243: 8 total, 7 up, 8 in 2024-08-17T17:31:18.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.930 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:31:18.930 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:18.961 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.966 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:31:18.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:19.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:31:19.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:19.037 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:31:19.037 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:19.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:19.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:31:19.075 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:19.078 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:19.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:19.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:19.116 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:31:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:19.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:19.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:19.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:19.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:19.510 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:19.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:19.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:19.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 81257 81241 73.5742 66.9375 0.0131092 0.013586 2024-08-17T17:31:19.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:19 smithi064 ceph-mon[104516]: osdmap e244: 8 total, 7 up, 8 in 2024-08-17T17:31:19.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4176151112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:19.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:19 smithi064 ceph-mon[109246]: osdmap e244: 8 total, 7 up, 8 in 2024-08-17T17:31:19.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4176151112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:19.842 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:19 smithi175 ceph-mon[101434]: osdmap e244: 8 total, 7 up, 8 in 2024-08-17T17:31:19.842 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4176151112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:19.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:19.899 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:19.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:19.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:20.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:20.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:20.104 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:20.104 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:20.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:20.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:20.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:20.298 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:20.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:20.372 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:20.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:20.474 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:20.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:20.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:20.647 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:20.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:20 smithi064 ceph-mon[104516]: pgmap v754: 89 pgs: 1 active+clean+remapped, 2 remapped+peering, 6 active+undersized+remapped, 22 active+undersized, 1 undersized+degraded+peered, 10 active+undersized+degraded, 47 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.1 MiB/s wr, 129 op/s; 39707/238494 objects degraded (16.649%); 8/238494 objects misplaced (0.003%); 224 KiB/s, 3 objects/s recovering 2024-08-17T17:31:20.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:20 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 39707/238494 objects degraded (16.649%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:20.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:20 smithi064 ceph-mon[109246]: pgmap v754: 89 pgs: 1 active+clean+remapped, 2 remapped+peering, 6 active+undersized+remapped, 22 active+undersized, 1 undersized+degraded+peered, 10 active+undersized+degraded, 47 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.1 MiB/s wr, 129 op/s; 39707/238494 objects degraded (16.649%); 8/238494 objects misplaced (0.003%); 224 KiB/s, 3 objects/s recovering 2024-08-17T17:31:20.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:20 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 39707/238494 objects degraded (16.649%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:20.749 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:20.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:20.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 82342 82326 73.4918 67.8125 0.00896872 0.0136013 2024-08-17T17:31:20.805 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:20 smithi175 ceph-mon[101434]: pgmap v754: 89 pgs: 1 active+clean+remapped, 2 remapped+peering, 6 active+undersized+remapped, 22 active+undersized, 1 undersized+degraded+peered, 10 active+undersized+degraded, 47 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.1 MiB/s wr, 129 op/s; 39707/238494 objects degraded (16.649%); 8/238494 objects misplaced (0.003%); 224 KiB/s, 3 objects/s recovering 2024-08-17T17:31:20.805 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:20 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 39707/238494 objects degraded (16.649%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:20.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:20.900 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:21.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:21.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:21.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:21.175 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:21.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.432 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:21.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:21.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:21.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:21.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:21.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 83469 83453 73.4487 70.4375 0.0160009 0.0136102 2024-08-17T17:31:21.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:21.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:21.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:21.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:22.148 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:22.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:22.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:22.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:22.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:22.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:22.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:22.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:22.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 84434 84418 73.2662 60.3125 0.0155794 0.0136435 2024-08-17T17:31:22.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:22.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:22 smithi175 ceph-mon[101434]: pgmap v755: 89 pgs: 7 active+clean+remapped, 1 remapped+peering, 1 active+undersized+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 25254/241131 objects degraded (10.473%); 8/241131 objects misplaced (0.003%); 436 KiB/s, 6 objects/s recovering 2024-08-17T17:31:22.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:22 smithi064 ceph-mon[104516]: pgmap v755: 89 pgs: 7 active+clean+remapped, 1 remapped+peering, 1 active+undersized+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 25254/241131 objects degraded (10.473%); 8/241131 objects misplaced (0.003%); 436 KiB/s, 6 objects/s recovering 2024-08-17T17:31:22.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:22 smithi064 ceph-mon[109246]: pgmap v755: 89 pgs: 7 active+clean+remapped, 1 remapped+peering, 1 active+undersized+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 25254/241131 objects degraded (10.473%); 8/241131 objects misplaced (0.003%); 436 KiB/s, 6 objects/s recovering 2024-08-17T17:31:22.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:23.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:23.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:23.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:23.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:23.363 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:23.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:23.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:23.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:23.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:23.632 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:31:23.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:23.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:23.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:23.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 85475 85459 73.1537 65.0625 0.0141682 0.013664 2024-08-17T17:31:23.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:23 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:23.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:23.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:23 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:23.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:23 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:23.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:24.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:24.091 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:24.160 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:24.161 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:24.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:24.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:23] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:31:24.239 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:24.341 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:24.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:24.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:24.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:24.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.653 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:24.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:24.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 86540 86524 73.0645 66.5625 0.0113432 0.0136807 2024-08-17T17:31:24.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:24.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:24 smithi175 ceph-mon[101434]: pgmap v756: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 25506/242598 objects degraded (10.514%); 7/242598 objects misplaced (0.003%); 368 KiB/s, 5 objects/s recovering 2024-08-17T17:31:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:24 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:24.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:24.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:24 smithi175 ceph-mon[101434]: osdmap e245: 8 total, 7 up, 8 in 2024-08-17T17:31:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[104516]: pgmap v756: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 25506/242598 objects degraded (10.514%); 7/242598 objects misplaced (0.003%); 368 KiB/s, 5 objects/s recovering 2024-08-17T17:31:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:24.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[104516]: osdmap e245: 8 total, 7 up, 8 in 2024-08-17T17:31:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[109246]: pgmap v756: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 293 op/s; 25506/242598 objects degraded (10.514%); 7/242598 objects misplaced (0.003%); 368 KiB/s, 5 objects/s recovering 2024-08-17T17:31:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/497957272' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:24 smithi064 ceph-mon[109246]: osdmap e245: 8 total, 7 up, 8 in 2024-08-17T17:31:25.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:25.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:25.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:31:25.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.179 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:25.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:25.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:25.302 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:25.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.450 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:25.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:25.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.596 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:25.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:25.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:25.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:25.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 87552 87536 72.9336 63.25 0.012441 0.0137057 2024-08-17T17:31:25.835 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2024-08-17T17:31:25.835 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:31:25.835 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2024-08-17T17:31:25.835 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2024-08-17T17:31:25.835 DEBUG:teuthology.orchestra.run.smithi175:osd.6> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T17:31:25.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl start ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:31:25.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:25.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:26.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.122 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:26.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:26.136 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:25 smithi175 systemd[1]: Starting Ceph osd.6 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:31:26.137 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-mon[101434]: pgmap v758: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 26809/255714 objects degraded (10.484%); 7/255714 objects misplaced (0.003%); 319 KiB/s, 4 objects/s recovering 2024-08-17T17:31:26.137 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:26.137 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:26.137 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 26809/255714 objects degraded (10.484%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:26.224 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:26.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:26.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:26.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:26.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:26.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.038812883 +0000 UTC m=+0.023038237 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:31:26.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.180581877 +0000 UTC m=+0.164807216 container create f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[104516]: pgmap v758: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 26809/255714 objects degraded (10.484%); 7/255714 objects misplaced (0.003%); 319 KiB/s, 4 objects/s recovering 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 26809/255714 objects degraded (10.484%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[109246]: pgmap v758: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 26809/255714 objects degraded (10.484%); 7/255714 objects misplaced (0.003%); 319 KiB/s, 4 objects/s recovering 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:26 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 26809/255714 objects degraded (10.484%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:26.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:26.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:26.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:26.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:26.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 88488 88472 72.7436 58.5 0.0744535 0.0137355 2024-08-17T17:31:26.857 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.484501507 +0000 UTC m=+0.468726850 container init f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:26.857 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.488824863 +0000 UTC m=+0.473050209 container start f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:31:26.857 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.59267532 +0000 UTC m=+0.576900673 container attach f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-17T17:31:26.870 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:26.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:27.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:27.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:27.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:27.161 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:31:27.162 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:31:27.163 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate[160551]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:31:27.163 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 bash[160501]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:31:27.163 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:26 smithi175 podman[160501]: 2024-08-17 17:31:26.926296204 +0000 UTC m=+0.910521556 container died f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-17T17:31:27.186 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:27.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:27.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:27.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:27.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:27.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:27.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:27.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:27.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 89382 89366 72.5245 55.875 0.00843684 0.0137828 2024-08-17T17:31:27.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:27.841 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:27 smithi175 podman[160722]: 2024-08-17 17:31:27.564345887 +0000 UTC m=+0.630972374 container remove f8e7466d08df2387e5675494d79079047d22779b7dd8e45d114f327bd3a61f2a (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:31:27.841 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:27 smithi175 podman[160799]: 2024-08-17 17:31:27.708209968 +0000 UTC m=+0.021022440 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:31:27.841 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:27 smithi175 podman[160799]: 2024-08-17 17:31:27.840722558 +0000 UTC m=+0.153535018 container create 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:31:27.877 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:27.979 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:28.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:28.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:28 smithi175 ceph-mon[101434]: pgmap v759: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 27089/258501 objects degraded (10.479%); 7/258501 objects misplaced (0.003%); 175 KiB/s, 2 objects/s recovering 2024-08-17T17:31:28.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.178 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.208 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:31:28.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:28.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:28.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 podman[160799]: 2024-08-17 17:31:28.102171973 +0000 UTC m=+0.414984444 container init 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T17:31:28.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 podman[160799]: 2024-08-17 17:31:28.106738612 +0000 UTC m=+0.419551077 container start 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-ced649f, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-08-17T17:31:28.412 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 ceph-osd[160834]: -- 172.21.15.175:0/1134734391 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55aa764351e0 con 0x55aa771b6400 2024-08-17T17:31:28.412 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 bash[160799]: 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e 2024-08-17T17:31:28.412 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 systemd[1]: Started Ceph osd.6 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:31:28.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:28 smithi064 ceph-mon[104516]: pgmap v759: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 27089/258501 objects degraded (10.479%); 7/258501 objects misplaced (0.003%); 175 KiB/s, 2 objects/s recovering 2024-08-17T17:31:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:28 smithi064 ceph-mon[109246]: pgmap v759: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 27089/258501 objects degraded (10.479%); 7/258501 objects misplaced (0.003%); 175 KiB/s, 2 objects/s recovering 2024-08-17T17:31:28.461 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:28.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:28.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:28.786 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 90328 90312 72.3527 59.125 0.00657911 0.0138158 2024-08-17T17:31:28.840 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:31:28.654+0000 7f112cea2640 -1 Falling back to public interface 2024-08-17T17:31:28.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:28.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:29.014 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:29.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:29.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:29.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:29.291 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:29.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:29.435 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:29.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:29.582 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:29.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:29.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:29.787 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 91214 91198 72.1377 55.375 0.00941422 0.0138572 2024-08-17T17:31:29.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:29.852 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:29.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:29.990 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:30.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.128 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:30.147 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.207 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:30 smithi175 ceph-mon[101434]: pgmap v760: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 27376/264852 objects degraded (10.336%); 7/264852 objects misplaced (0.003%); 147 KiB/s, 1 objects/s recovering 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:no valid command found; 10 closest matches: 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:0 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:1 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:2 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:abort 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:assert 2024-08-17T17:31:30.226 INFO:teuthology.orchestra.run.smithi175.stderr:config diff 2024-08-17T17:31:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:config diff get 2024-08-17T17:31:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:config get 2024-08-17T17:31:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:config help [] 2024-08-17T17:31:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:config set ... 2024-08-17T17:31:30.227 INFO:teuthology.orchestra.run.smithi175.stderr:admin_socket: invalid command 2024-08-17T17:31:30.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:30.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:30.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:30.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:30 smithi064 ceph-mon[104516]: pgmap v760: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 27376/264852 objects degraded (10.336%); 7/264852 objects misplaced (0.003%); 147 KiB/s, 1 objects/s recovering 2024-08-17T17:31:30.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:30 smithi064 ceph-mon[109246]: pgmap v760: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 27376/264852 objects degraded (10.336%); 7/264852 objects misplaced (0.003%); 147 KiB/s, 1 objects/s recovering 2024-08-17T17:31:30.422 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:30.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:30.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:30.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:30.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:30.787 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:31:30.786960+0000 min lat: 0.00120552 max lat: 17.1092 avg lat: 0.0138854 2024-08-17T17:31:30.787 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:30.787 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 92174 92158 71.9859 60 0.0182526 0.0138854 2024-08-17T17:31:30.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:30.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:31.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:31:31.075 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2024-08-17T17:31:31.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:31.123 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:31:31.123 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:31.162 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:31:31.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:31.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:31.217 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:31:31.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:31.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:31.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:31:31.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:31.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:31 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28288/271239 objects degraded (10.429%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:31.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/273181143' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:31.298 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:31:31.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:31.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:31 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28288/271239 objects degraded (10.429%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:31.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/273181143' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:31.332 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:31:31.332 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:31.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:31.356 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:31.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:31:31.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:31.382 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:31.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:31:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:31 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28288/271239 objects degraded (10.429%), 7 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/273181143' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:31.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:31.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:31.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:31.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:31.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:31.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:31.793 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 93008 92992 71.7382 52.125 0.0307854 0.0139339 2024-08-17T17:31:31.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:31.921 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:31.945 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:32.047 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:32.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:32.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:32.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:32.311 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:32.312 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:32.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:32.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:32 smithi175 ceph-mon[101434]: pgmap v761: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 28288/271239 objects degraded (10.429%); 7/271239 objects misplaced (0.003%) 2024-08-17T17:31:32.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:32 smithi064 ceph-mon[104516]: pgmap v761: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 28288/271239 objects degraded (10.429%); 7/271239 objects misplaced (0.003%) 2024-08-17T17:31:32.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:32 smithi064 ceph-mon[109246]: pgmap v761: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 28288/271239 objects degraded (10.429%); 7/271239 objects misplaced (0.003%) 2024-08-17T17:31:32.445 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:32.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:32.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:32.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:32.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:32.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:32.792 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 93935 93919 71.5681 57.9375 0.0230992 0.0139666 2024-08-17T17:31:32.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:32.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:32.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:32.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:33.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:33.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:33.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:33.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:33.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:33.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:33.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:33.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:33.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:33.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:33.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:33.741 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:31:33.742 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:31:33.742 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:31:33.742 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:31:33.742 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:31:33.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:33.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:33.792 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 94792 94776 71.3511 53.5625 0.0112265 0.0140094 2024-08-17T17:31:33.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:33.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:33.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:34.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:34.118 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:34.132 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:34 smithi175 ceph-mon[101434]: pgmap v762: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 28545/272871 objects degraded (10.461%); 7/272871 objects misplaced (0.003%) 2024-08-17T17:31:34.132 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:34 smithi175 ceph-mon[101434]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.132 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:34 smithi175 ceph-mon[101434]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.132 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:31:33.750+0000 7f112cea2640 -1 osd.6 199 log_to_monitors true 2024-08-17T17:31:34.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[104516]: pgmap v762: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 28545/272871 objects degraded (10.461%); 7/272871 objects misplaced (0.003%) 2024-08-17T17:31:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[104516]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[104516]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:33] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.51.0" 2024-08-17T17:31:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[109246]: pgmap v762: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 28545/272871 objects degraded (10.461%); 7/272871 objects misplaced (0.003%) 2024-08-17T17:31:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[109246]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:34 smithi064 ceph-mon[109246]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:31:34.244 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:34.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:34.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:31:34.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:34.398 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:34.410 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:31:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:31:34.286+0000 7f1124c4c640 -1 osd.6 246 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:31:34.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:34.540 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:34.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:34.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:34.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:34.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 95740 95724 71.2054 59.25 0.0143944 0.0140387 2024-08-17T17:31:34.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:34.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:34.960 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:34.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:35.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:35.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:35.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:35.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:35.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:35.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:31:35.350 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:35 smithi175 ceph-mon[101434]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:31:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:35 smithi175 ceph-mon[101434]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:35 smithi175 ceph-mon[101434]: osdmap e246: 8 total, 7 up, 8 in 2024-08-17T17:31:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:35 smithi175 ceph-mon[101434]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[109246]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:31:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[109246]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[109246]: osdmap e246: 8 total, 7 up, 8 in 2024-08-17T17:31:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[109246]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[104516]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:31:35.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[104516]: from='osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[104516]: osdmap e246: 8 total, 7 up, 8 in 2024-08-17T17:31:35.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:35 smithi064 ceph-mon[104516]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:31:35.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:35.474 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:35.576 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:35.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:35.700 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:35.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:35.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 96137 96121 70.6596 24.8125 0.00346278 0.0140472 2024-08-17T17:31:35.823 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:35.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:35.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:35.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:35.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:36.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:36.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:31:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:36.099 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:36.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:36.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:36.304 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:36.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:36.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:36.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: pgmap v764: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 29631/283650 objects degraded (10.446%); 7/283650 objects misplaced (0.002%) 2024-08-17T17:31:36.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:36.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060] boot 2024-08-17T17:31:36.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[104516]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: pgmap v764: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 29631/283650 objects degraded (10.446%); 7/283650 objects misplaced (0.002%) 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060] boot 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:31:36.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:36 smithi064 ceph-mon[109246]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: pgmap v764: 89 pgs: 9 active+clean+remapped, 16 active+undersized, 7 active+undersized+degraded, 57 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 29631/283650 objects degraded (10.446%); 7/283650 objects misplaced (0.002%) 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: osd.6 [v2:172.21.15.175:6816/1826486060,v1:172.21.15.175:6817/1826486060] boot 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:31:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:36 smithi175 ceph-mon[101434]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:31:36.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:36.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:36.471 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:36.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:36.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:36.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:36.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:36.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 96137 96121 69.8381 0 - 0.0140472 2024-08-17T17:31:36.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:36.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:36.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:36.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:37.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:37.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:37.147 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:31:37.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:37.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:37.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:37.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:37 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:37 smithi175 ceph-mon[101434]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:31:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[104516]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2610039829' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:37 smithi064 ceph-mon[109246]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:31:37.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:37.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:37.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:37.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:37.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 96436 96420 69.2502 9.34375 0.0213107 0.0144344 2024-08-17T17:31:37.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.906 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:37.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:38.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:38.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:38.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:38.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:38.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:38.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:38.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:38 smithi175 ceph-mon[101434]: pgmap v767: 89 pgs: 7 remapped+peering, 4 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 64 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 23916/285852 objects degraded (8.367%); 4/285852 objects misplaced (0.001%) 2024-08-17T17:31:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23916/285852 objects degraded (8.367%), 4 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[104516]: pgmap v767: 89 pgs: 7 remapped+peering, 4 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 64 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 23916/285852 objects degraded (8.367%); 4/285852 objects misplaced (0.001%) 2024-08-17T17:31:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23916/285852 objects degraded (8.367%), 4 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[109246]: pgmap v767: 89 pgs: 7 remapped+peering, 4 peering, 2 active+clean+remapped, 8 active+undersized, 4 active+undersized+degraded, 64 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 23916/285852 objects degraded (8.367%); 4/285852 objects misplaced (0.001%) 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23916/285852 objects degraded (8.367%), 4 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:38.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:38.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:38.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:38.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:38.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:38.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:38.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:38.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 97238 97222 69.0327 50.125 0.0165253 0.0144798 2024-08-17T17:31:38.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:38.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:39.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.073 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:39.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:39.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:39.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:39.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:39.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:39.449 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:31:39.449 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:31:39.449 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.449 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:97742 7.b e249/248 7:d443faf8:::benchmark_data_smithi064_210574_object97741:head v 249'6161, mlcod=0'0)", 2024-08-17T17:31:39.449 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:39.425022+0000", 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.015328736000000001, 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015360331, 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425022+0000", 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425022+0000", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425020+0000", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425070+0000", 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9339000000000003e-05 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425090+0000", 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0809999999999999e-05 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425099+0000", 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0170000000000002e-06 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:31:39.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425351+0000", 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025136700000000001 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.425430+0000", 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9104999999999996e-05 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:97734 7.1 e249/248 7:83364941:::benchmark_data_smithi064_210574_object97733:head v 249'6024, mlcod=233'58)", 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:39.436140+0000", 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0042102320000000004, 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043004799999999998, 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436140+0000", 2024-08-17T17:31:39.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436140+0000", 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436138+0000", 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436531+0000", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00039261299999999999 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436557+0000", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6492999999999999e-05 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436568+0000", 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0529e-05 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436633+0000", 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5518000000000005e-05 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436724+0000", 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0264000000000002e-05 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.457 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:97749 7.d e249/248 7:b797e070:::benchmark_data_smithi064_210574_object97748:head v 249'6105, mlcod=0'0)", 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:39.436856+0000", 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0034940129999999998, 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036323950000000001, 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436856+0000", 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436856+0000", 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436855+0000", 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:31:39.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436889+0000", 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.341e-05 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436901+0000", 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1851999999999999e-05 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.436907+0000", 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.657e-06 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.437863+0000", 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00095551800000000004 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:39.437890+0000", 2024-08-17T17:31:39.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7345e-05 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 3 2024-08-17T17:31:39.462 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:31:39.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:39.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:39.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.794 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 98049 98033 68.8266 50.6875 0.0101228 0.0145232 2024-08-17T17:31:39.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:39.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:40.158 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:40.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:40.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:40.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:40.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:40.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:40 smithi175 ceph-mon[101434]: pgmap v769: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 4 peering, 2 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 65 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 24005/289077 objects degraded (8.304%); 4/289077 objects misplaced (0.001%) 2024-08-17T17:31:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[109246]: pgmap v769: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 4 peering, 2 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 65 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 24005/289077 objects degraded (8.304%); 4/289077 objects misplaced (0.001%) 2024-08-17T17:31:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[104516]: pgmap v769: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 4 peering, 2 active+clean+remapped, 7 active+undersized, 3 active+undersized+degraded, 65 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 24005/289077 objects degraded (8.304%); 4/289077 objects misplaced (0.001%) 2024-08-17T17:31:40.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:40.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:40.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:40.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:40.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.786 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:40.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:40.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 90 3 98806 98803 68.5965 48.125 0.0236109 0.014572 2024-08-17T17:31:40.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:40.909 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:41.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:41.032 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:131941 5.0 5:0bbb7707:::benchmark_data_smithi064_147470_object131940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 578.95919542499996, 2024-08-17T17:31:41.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.435185814, 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 131941 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083754+0000", 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083756+0000", 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9989999999999998e-06 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083779+0000", 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2419999999999999e-05 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083798+0000", 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8740999999999999e-05 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083805+0000", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6480000000000003e-06 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083928+0000", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012243700000000001 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:02.083942+0000", 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3985000000000001e-05 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.047013+0000", 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96307144200000006 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:03.047435+0000", 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042231199999999999 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.498588+0000", 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4511525169999999 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.498745+0000", 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015708900000000001 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.499521+0000", 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00077622900000000002 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.513675+0000", 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014153361999999999 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.513798+0000", 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012309899999999999 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515148+0000", 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013501419999999999 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.515229+0000", 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0810999999999996e-05 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:04.518940+0000", 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037115809999999998 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140798 5.0 5:06e39447:::benchmark_data_smithi064_147470_object8854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 560.78842052200002, 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1158812280000001, 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140798 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254529+0000", 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254531+0000", 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.499e-06 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254537+0000", 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5719999999999999e-06 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254555+0000", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7589999999999999e-05 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254563+0000", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2360000000000004e-06 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254638+0000", 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4779999999999999e-05 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.254651+0000", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2469e-05 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.265920+0000", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.011269057 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.266255+0000", 2024-08-17T17:31:41.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033558699999999998 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.266541+0000", 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028578399999999997 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.274465+0000", 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079242489999999995 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370009+0000", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.095543367000000004 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370205+0000", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195995 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370268+0000", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3637000000000001e-05 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370411+0000", 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014240600000000001 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:140799 5.0 5:03f574a7:::benchmark_data_smithi064_147470_object8855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 560.78276257899995, 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.114930542, 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.061 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140799 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260187+0000", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260189+0000", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.393e-06 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260193+0000", 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.702e-06 2024-08-17T17:31:41.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260202+0000", 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4330000000000004e-06 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.260209+0000", 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3830000000000003e-06 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.273511+0000", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013301453 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:20.273522+0000", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0890000000000001e-05 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271047+0000", 2024-08-17T17:31:41.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99752509700000003 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271196+0000", 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014981400000000001 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.271334+0000", 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013721299999999999 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370162+0000", 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098828572000000003 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.370457+0000", 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029438799999999997 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374859+0000", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044020370000000001 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.374920+0000", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.156e-05 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:21.375118+0000", 2024-08-17T17:31:41.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019760700000000001 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:151624 5.0 5:015e5863:::benchmark_data_smithi064_147470_object19680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:39.289697+0000", 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 541.75325279699996, 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.989650845, 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.069 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151624 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289697+0000", 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289697+0000", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289699+0000", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8810000000000001e-06 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289705+0000", 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1360000000000002e-06 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289725+0000", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0069e-05 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289734+0000", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5960000000000004e-06 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289841+0000", 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107014 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.289853+0000", 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1928e-05 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.264372+0000", 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97451896000000005 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.264503+0000", 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013178300000000001 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.264632+0000", 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012826100000000001 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.272293+0000", 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076609929999999996 2024-08-17T17:31:41.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279251+0000", 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006958428 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279272+0000", 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1022e-05 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279316+0000", 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3993999999999999e-05 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279348+0000", 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1779999999999997e-05 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:151641 5.0 5:020a4cc7:::benchmark_data_smithi064_147470_object19697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:39.346276+0000", 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 541.69667373699997, 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94175968600000004, 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.078 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151641 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346276+0000", 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346276+0000", 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346278+0000", 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7540000000000001e-06 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346284+0000", 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5230000000000001e-06 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346307+0000", 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2171000000000002e-05 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346316+0000", 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346398+0000", 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1985999999999997e-05 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.346412+0000", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.429e-05 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.268482+0000", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92206967200000001 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.268604+0000", 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012253000000000001 2024-08-17T17:31:41.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.268744+0000", 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013920600000000001 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.272346+0000", 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036027250000000002 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279449+0000", 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071031690000000003 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.287928+0000", 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008478289 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.287983+0000", 2024-08-17T17:31:41.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4829e-05 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288036+0000", 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3270000000000003e-05 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:151662 5.0 5:052cb3ba:::benchmark_data_smithi064_147470_object19718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:39.410923+0000", 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 541.632026774, 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87720358799999998, 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151662 2024-08-17T17:31:41.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410923+0000", 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410923+0000", 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410925+0000", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.753e-06 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410932+0000", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3710000000000002e-06 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410953+0000", 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1090999999999998e-05 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.410961+0000", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5739999999999998e-06 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.411039+0000", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8101000000000003e-05 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.411052+0000", 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2819e-05 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.272415+0000", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86136323199999998 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.272597+0000", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000181559 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.272746+0000", 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014933700000000001 2024-08-17T17:31:41.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279397+0000", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066511369999999997 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.287911+0000", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085138339999999996 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288049+0000", 2024-08-17T17:31:41.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013777099999999999 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288090+0000", 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1712999999999999e-05 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288127+0000", 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6295999999999999e-05 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34626.0:151685 5.0 5:062c395d:::benchmark_data_smithi064_147470_object19741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:22:39.467130+0000", 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 541.57581943599996, 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.82119465899999999, 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34626", 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3769576205", 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151685 2024-08-17T17:31:41.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467130+0000", 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467130+0000", 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467132+0000", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0279999999999999e-06 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467139+0000", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3640000000000004e-06 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467155+0000", 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6444e-05 2024-08-17T17:31:41.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467164+0000", 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0250000000000008e-06 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467240+0000", 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5284000000000006e-05 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:39.467252+0000", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2381000000000001e-05 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.276179+0000", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80892719899999999 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.276304+0000", 2024-08-17T17:31:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012468099999999999 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.276439+0000", 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013506799999999999 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.279591+0000", 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003151671 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288137+0000", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008546517 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288231+0000", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3857999999999995e-05 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288283+0000", 2024-08-17T17:31:41.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2172999999999997e-05 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:22:40.288325+0000", 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1965999999999998e-05 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70096 6.11 6.2cfda691 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e181)", 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 285.81914882199999, 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98497944299999995, 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.101 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70096 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223802+0000", 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.381e-06 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223826+0000", 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3985000000000002e-05 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223836+0000", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5440000000000004e-06 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223842+0000", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.224625+0000", 2024-08-17T17:31:41.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00078297999999999996 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.224631+0000", 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559999999999997e-06 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208765+0000", 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98413317700000003 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208781+0000", 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.592e-05 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70132 6.11 6.2d46dcd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e181)", 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 285.78188679800002, 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94775391899999994, 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.106 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70132 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261065+0000", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579999999999999e-06 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261099+0000", 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4001000000000002e-05 2024-08-17T17:31:41.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261122+0000", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2575999999999999e-05 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261133+0000", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0815999999999999e-05 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.262936+0000", 2024-08-17T17:31:41.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001802943 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.262944+0000", 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8400000000000001e-06 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208810+0000", 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94586556700000002 2024-08-17T17:31:41.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208817+0000", 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0180000000000004e-06 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71363 6.13 6.18c43493 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 281.76528162900001, 2024-08-17T17:31:41.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86920691299999997, 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71363 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:31:41.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277669+0000", 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.049e-06 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277719+0000", 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9466999999999999e-05 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277729+0000", 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8900000000000002e-06 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277735+0000", 2024-08-17T17:31:41.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5549999999999999e-06 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.278009+0000", 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000273873 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.278017+0000", 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146862+0000", 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86884535299999999 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146875+0000", 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3327e-05 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.116 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71379 6.13 6.a90a54b3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 281.75101636800002, 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85498798300000001, 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71379 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.117 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0500000000000003e-07 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292001+0000", 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7251999999999999e-05 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292012+0000", 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0624e-05 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292019+0000", 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5010000000000001e-06 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292064+0000", 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5040000000000002e-05 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292069+0000", 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4299999999999997e-06 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146913+0000", 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85484384199999996 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146922+0000", 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6889999999999993e-06 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71386 6.13 6.fb4201f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 281.74364124800002, 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.847643182, 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.122 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71386 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299310+0000", 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1400000000000001e-06 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299334+0000", 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4596e-05 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299344+0000", 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6299999999999993e-06 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299350+0000", 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.300057+0000", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070641699999999998 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.300065+0000", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4710000000000007e-06 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146945+0000", 2024-08-17T17:31:41.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.846879402 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146952+0000", 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1210000000000001e-06 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71399 6.13 6.336b4e73 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:31:41.127 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 281.73491142699999, 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83894056100000003, 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71399 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308040+0000", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0389999999999999e-06 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308093+0000", 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3161000000000001e-05 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308105+0000", 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2031000000000001e-05 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308112+0000", 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308165+0000", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2988999999999997e-05 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308172+0000", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8460000000000002e-06 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146973+0000", 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83880076199999998 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146979+0000", 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1330000000000004e-06 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101312 6.c 6:30f79fae:::benchmark_data_smithi064_185391_object101311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 245.86699305100001, 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0150590660000001, 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.132 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101312 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175958+0000", 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243e-06 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175982+0000", 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3527e-05 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175992+0000", 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0873000000000001e-05 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176000+0000", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176053+0000", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3437999999999999e-05 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176059+0000", 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7690000000000002e-06 2024-08-17T17:31:41.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169698+0000", 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99363913199999998 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169828+0000", 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012921800000000001 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169985+0000", 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157904 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190558+0000", 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020572724000000001 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190868+0000", 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031017000000000002 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190931+0000", 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2501000000000003e-05 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191016+0000", 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5037999999999995e-05 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.139 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101316 6.c 6:3bdd1909:::benchmark_data_smithi064_185391_object101315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 245.864208985, 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0124066309999999, 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.140 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101316 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178742+0000", 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1340000000000001e-06 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178764+0000", 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2075000000000001e-05 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178776+0000", 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1766999999999999e-05 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178783+0000", 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7510000000000002e-06 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178897+0000", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114405 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178905+0000", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9510000000000006e-06 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175170+0000", 2024-08-17T17:31:41.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99626462900000001 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175365+0000", 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195873 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175512+0000", 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014636 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190619+0000", 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015107318999999999 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191035+0000", 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000416177 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191076+0000", 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0827000000000001e-05 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191148+0000", 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1363000000000001e-05 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101345 6.c 6:3c451189:::benchmark_data_smithi064_185391_object101344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:31:41.147 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 245.83237995600001, 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98069468500000001, 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101345 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210571+0000", 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.212e-06 2024-08-17T17:31:41.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210601+0000", 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0018999999999998e-05 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210619+0000", 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7586999999999999e-05 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210626+0000", 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9369999999999998e-06 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210696+0000", 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0623000000000005e-05 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210705+0000", 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1479999999999995e-06 2024-08-17T17:31:41.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178984+0000", 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96827804699999998 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.179094+0000", 2024-08-17T17:31:41.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110531 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.179234+0000", 2024-08-17T17:31:41.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014043899999999999 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190670+0000", 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01143533 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191155+0000", 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048500700000000003 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191194+0000", 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9084000000000001e-05 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191265+0000", 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0721000000000003e-05 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.156 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101356 6.c 6:30433116:::benchmark_data_smithi064_185391_object101355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 245.81667221199999, 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96510339099999998, 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101356 2024-08-17T17:31:41.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226279+0000", 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3e-06 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226302+0000", 2024-08-17T17:31:41.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2875e-05 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226316+0000", 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4109e-05 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226324+0000", 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5519999999999999e-06 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226388+0000", 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4151000000000003e-05 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226397+0000", 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.533e-06 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183269+0000", 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95687179300000003 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183388+0000", 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011864199999999999 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183529+0000", 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014124299999999999 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190713+0000", 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071838409999999998 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191271+0000", 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055812600000000004 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191310+0000", 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9085000000000003e-05 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191381+0000", 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1141000000000002e-05 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101360 6.c 6:3d93d054:::benchmark_data_smithi064_185391_object101359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 245.81185935100001, 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96101782800000002, 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101360 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231092+0000", 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173e-06 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231111+0000", 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9038999999999998e-05 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231122+0000", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1586000000000001e-05 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231132+0000", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2539999999999998e-06 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231190+0000", 2024-08-17T17:31:41.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8338000000000003e-05 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231198+0000", 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.678e-06 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.186989+0000", 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95579149600000002 2024-08-17T17:31:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187130+0000", 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014124799999999999 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187293+0000", 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016258700000000001 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190766+0000", 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003472709 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191911+0000", 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011457920000000001 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.192030+0000", 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118986 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.192108+0000", 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7941999999999993e-05 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.172 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96080 7.c 7:3f226a37:::benchmark_data_smithi064_210574_object96079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.8973971780000003, 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2216669489999998, 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96080 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:31:41.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145554+0000", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2600000000000001e-07 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145578+0000", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4579999999999998e-05 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145587+0000", 2024-08-17T17:31:41.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5979999999999997e-06 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145593+0000", 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7620000000000001e-06 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145700+0000", 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010727500000000001 2024-08-17T17:31:41.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145831+0000", 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013128200000000001 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145967+0000", 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013590600000000001 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163850+0000", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017883087999999998 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163863+0000", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3019e-05 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149735+0000", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98587233299999999 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149749+0000", 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3543999999999999e-05 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367152+0000", 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217402664 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367220+0000", 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7972000000000006e-05 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96087 7.c 7:347c5818:::benchmark_data_smithi064_210574_object96086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.8918874460000001, 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.232266434, 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:31:41.181 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96087 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151064+0000", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6819999999999999e-06 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151095+0000", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0735999999999998e-05 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151106+0000", 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1232e-05 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151129+0000", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2767000000000001e-05 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163899+0000", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012769951 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163907+0000", 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.354e-06 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149776+0000", 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98586885099999999 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149784+0000", 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2039999999999997e-06 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:31:41.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370442+0000", 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.220658105 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370625+0000", 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018222600000000001 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370817+0000", 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019209699999999999 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.373489+0000", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026719090000000001 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.375180+0000", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016911890000000001 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380081+0000", 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049010909999999998 2024-08-17T17:31:41.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380117+0000", 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6176000000000001e-05 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.383329+0000", 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003211864 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:31:41.189 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:31:41.189 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:31:41.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:41.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:41.444 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:41.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:41.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:41.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:41.718 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0265 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 98806 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 68.5951 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 45.191 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 192.812 2024-08-17T17:31:41.795 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1097 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 723.072 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3085 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0145726 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.208891 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 17.1092 2024-08-17T17:31:41.796 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00120552 2024-08-17T17:31:41.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:31:41.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:42.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:42.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:42.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:31:42.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:31:42.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:31:42.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:31:42.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:42.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:42.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:42.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:42.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:42.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:42.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:42.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:42 smithi064 ceph-mon[104516]: pgmap v770: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 79 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s; 24121/292233 objects degraded (8.254%) 2024-08-17T17:31:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3696093114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:42 smithi064 ceph-mon[109246]: pgmap v770: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 79 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s; 24121/292233 objects degraded (8.254%) 2024-08-17T17:31:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3696093114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:42.472 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:42 smithi175 ceph-mon[101434]: pgmap v770: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 79 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s; 24121/292233 objects degraded (8.254%) 2024-08-17T17:31:42.473 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3696093114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:42.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:42.542 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:42.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:42.672 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:42.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:42.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:42.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:42.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:43.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:43.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:43.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:43.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:43.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:43.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:43.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:43.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:43.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:43.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:43.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:43.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:43.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:43.649 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29909/293235 objects degraded (10.200%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29909/293235 objects degraded (10.200%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:43.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29909/293235 objects degraded (10.200%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:43.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:43.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:43.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:43.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:44.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:44.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:44.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:44.156 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:43] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-08-17T17:31:44.190 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:44.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:44.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:44.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:44.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.564 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:44 smithi175 ceph-mon[101434]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 29909/293235 objects degraded (10.200%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:31:44.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:44.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:44 smithi064 ceph-mon[104516]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 29909/293235 objects degraded (10.200%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:31:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:44 smithi064 ceph-mon[109246]: pgmap v771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 29909/293235 objects degraded (10.200%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:31:44.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:44.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:44.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:44.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:44.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:44.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:31:45.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:45.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:45.186 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:31:45.186 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:31:45.187 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2024-08-17T17:31:45.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:31:45.275 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:45.318 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:45.332 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:45.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:45.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:45.534 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:45.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:45.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:45.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:45.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:45.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:45.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:46.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:46.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[109246]: pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 27 MiB/s wr, 591 op/s; 29909/292023 objects degraded (10.242%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:31:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:46.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:45 smithi064 ceph-mon[104516]: pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 27 MiB/s wr, 591 op/s; 29909/292023 objects degraded (10.242%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:31:46.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:46.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:45 smithi175 ceph-mon[101434]: pgmap v772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 27 MiB/s wr, 591 op/s; 29909/292023 objects degraded (10.242%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:31:46.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:46.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:46.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:46.353 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:46.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:46.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:46.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:46.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:46.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:46.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:46.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:46.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:47.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:47.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:47.242 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:47.243 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:47.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:47.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:47.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:47.532 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:47.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:47.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:47.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:47.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:47.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:47.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:31:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:48.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:48 smithi064 ceph-mon[104516]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 567 op/s; 29754/290145 objects degraded (10.255%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:48.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:48.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:48 smithi175 ceph-mon[101434]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 567 op/s; 29754/290145 objects degraded (10.255%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:48.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:48.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:48.257 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:48.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:48.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:48.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:48 smithi064 ceph-mon[109246]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 567 op/s; 29754/290145 objects degraded (10.255%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:48.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:48.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:48.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:48.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:48.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:48.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:48.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:48.953 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:49.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:49.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:49.195 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:49.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:49.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:49.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:49 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29754/290145 objects degraded (10.255%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:49 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29754/290145 objects degraded (10.255%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:49.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:49 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29754/290145 objects degraded (10.255%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:49.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:49.490 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:49.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:49.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:49.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:49.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:49.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:50.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:50.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:50.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:50.123 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:50.123 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:50.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:50.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:50.249 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:31:50.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:50.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:50 smithi064 ceph-mon[104516]: pgmap v774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 B/s rd, 19 MiB/s wr, 625 op/s; 29754/284868 objects degraded (10.445%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:31:50.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:50.319 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:50 smithi064 ceph-mon[109246]: pgmap v774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 B/s rd, 19 MiB/s wr, 625 op/s; 29754/284868 objects degraded (10.445%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:31:50.319 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:50.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:50.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:50 smithi175 ceph-mon[101434]: pgmap v774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 B/s rd, 19 MiB/s wr, 625 op/s; 29754/284868 objects degraded (10.445%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:31:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:50.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:50.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:50.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:50.612 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:50.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:50.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:31:50.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:50.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:50.977 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:31:50.983 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:31:51.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:51.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:51.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:51.170 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:51.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:51.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:51.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:51.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:51.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[104516]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:31:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:51 smithi064 ceph-mon[109246]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:31:51.445 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:51 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:51.445 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:51 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2685586774' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:51.445 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:51 smithi175 ceph-mon[101434]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:31:51.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:51.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:51.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:51.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:51.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:51.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:51.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:52.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:52.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:52.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:52.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:52.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:52.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:52.317 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":775,"stamp":"2024-08-17T17:31:50.252251+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6129129469,"num_objects":93479,"num_object_clones":0,"num_object_copies":280437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93479,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":105286,"num_write_kb":6338626,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":32251715,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105345,"ondisk_log_size":105345,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":282,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21229972,"kb_used_data":18392148,"kb_used_omap":46,"kb_used_meta":2837393,"kb_avail":728731244,"statfs":{"total":767960285184,"available":746220793856,"internally_reserved":0,"allocated":18833559552,"data_stored":18822596306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":2905490675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-188809192,"num_objects":-2880,"num_object_clones":0,"num_object_copies":-8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5749,"num_objects_misplaced":-4,"num_objects_unfound":0,"num_objects_dirty":-2880,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7832,"num_write_kb":158401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.235331"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":354,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150272+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:31:37.150272+0000","last_peered":"2024-08-17T17:31:37.150272+0000","last_clean":"2024-08-17T17:31:37.150272+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:31:37.150272+0000","last_undegraded":"2024-08-17T17:31:37.150272+0000","last_fullsized":"2024-08-17T17:31:37.150272+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.150001+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:31:36.150001+0000","last_peered":"2024-08-17T17:31:36.150001+0000","last_clean":"2024-08-17T17:31:36.150001+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:31:36.150001+0000","last_undegraded":"2024-08-17T17:31:36.150001+0000","last_fullsized":"2024-08-17T17:31:36.150001+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":29,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.148052+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:31:36.148052+0000","last_peered":"2024-08-17T17:31:36.148052+0000","last_clean":"2024-08-17T17:31:36.148052+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:31:36.148052+0000","last_undegraded":"2024-08-17T17:31:36.148052+0000","last_fullsized":"2024-08-17T17:31:36.148052+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":346,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.152920+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:31:37.152920+0000","last_peered":"2024-08-17T17:31:37.152920+0000","last_clean":"2024-08-17T17:31:37.152920+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:31:37.152920+0000","last_undegraded":"2024-08-17T17:31:37.152920+0000","last_fullsized":"2024-08-17T17:31:37.152920+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":346,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153522+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:31:37.153522+0000","last_peered":"2024-08-17T17:31:37.153522+0000","last_clean":"2024-08-17T17:31:37.153522+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:31:37.153522+0000","last_undegraded":"2024-08-17T17:31:37.153522+0000","last_fullsized":"2024-08-17T17:31:37.153522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":231,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.163531+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:31:36.163531+0000","last_peered":"2024-08-17T17:31:36.163531+0000","last_clean":"2024-08-17T17:31:36.163531+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:31:36.163531+0000","last_undegraded":"2024-08-17T17:31:36.163531+0000","last_fullsized":"2024-08-17T17:31:36.163531+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":354,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.154040+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:31:37.154040+0000","last_peered":"2024-08-17T17:31:37.154040+0000","last_clean":"2024-08-17T17:31:37.154040+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:31:37.154040+0000","last_undegraded":"2024-08-17T17:31:37.154040+0000","last_fullsized":"2024-08-17T17:31:37.154040+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":179,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.159285+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:31:37.159285+0000","last_peered":"2024-08-17T17:31:37.159285+0000","last_clean":"2024-08-17T17:31:37.159285+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:31:37.159285+0000","last_undegraded":"2024-08-17T17:31:37.159285+0000","last_fullsized":"2024-08-17T17:31:37.159285+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"249'6632","reported_seq":6692,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:49.687507+0000","last_change":"2024-08-17T17:31:37.408577+0000","last_active":"2024-08-17T17:31:49.687507+0000","last_peered":"2024-08-17T17:31:49.687507+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.396040+0000","last_became_peered":"2024-08-17T17:31:37.396040+0000","last_unstale":"2024-08-17T17:31:49.687507+0000","last_undegraded":"2024-08-17T17:31:37.194642+0000","last_fullsized":"2024-08-17T17:31:37.151313+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381288448,"num_objects":5818,"num_object_clones":0,"num_object_copies":17454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6632,"num_write_kb":398400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-245}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":240,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.152941+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:31:36.152941+0000","last_peered":"2024-08-17T17:31:36.152941+0000","last_clean":"2024-08-17T17:31:36.152941+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:31:36.152941+0000","last_undegraded":"2024-08-17T17:31:36.152941+0000","last_fullsized":"2024-08-17T17:31:36.152941+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.156990+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:31:37.156990+0000","last_peered":"2024-08-17T17:31:37.156990+0000","last_clean":"2024-08-17T17:31:37.156990+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:31:37.156990+0000","last_undegraded":"2024-08-17T17:31:37.156990+0000","last_fullsized":"2024-08-17T17:31:37.156990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"249'6483","reported_seq":6581,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.380993+0000","last_change":"2024-08-17T17:31:17.798927+0000","last_active":"2024-08-17T17:31:49.380993+0000","last_peered":"2024-08-17T17:31:49.380993+0000","last_clean":"2024-08-17T17:31:49.380993+0000","last_became_active":"2024-08-17T17:31:17.603128+0000","last_became_peered":"2024-08-17T17:31:17.603128+0000","last_unstale":"2024-08-17T17:31:49.380993+0000","last_undegraded":"2024-08-17T17:31:49.380993+0000","last_fullsized":"2024-08-17T17:31:49.380993+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6483,"log_dups_size":0,"ondisk_log_size":6483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:03.029811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379518976,"num_objects":5791,"num_object_clones":0,"num_object_copies":17373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6483,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":241,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.152504+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:31:36.152504+0000","last_peered":"2024-08-17T17:31:36.152504+0000","last_clean":"2024-08-17T17:31:36.152504+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:31:36.152504+0000","last_undegraded":"2024-08-17T17:31:36.152504+0000","last_fullsized":"2024-08-17T17:31:36.152504+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.155872+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:31:36.155872+0000","last_peered":"2024-08-17T17:31:36.155872+0000","last_clean":"2024-08-17T17:31:36.155872+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:31:36.155872+0000","last_undegraded":"2024-08-17T17:31:36.155872+0000","last_fullsized":"2024-08-17T17:31:36.155872+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"249'6640","reported_seq":6690,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.892275+0000","last_change":"2024-08-17T17:29:57.202283+0000","last_active":"2024-08-17T17:31:48.892275+0000","last_peered":"2024-08-17T17:31:48.892275+0000","last_clean":"2024-08-17T17:31:48.892275+0000","last_became_active":"2024-08-17T17:29:57.201797+0000","last_became_peered":"2024-08-17T17:29:57.201797+0000","last_unstale":"2024-08-17T17:31:48.892275+0000","last_undegraded":"2024-08-17T17:31:48.892275+0000","last_fullsized":"2024-08-17T17:31:48.892275+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6640,"log_dups_size":0,"ondisk_log_size":6640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:53.663893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":258,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.156907+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:31:37.156907+0000","last_peered":"2024-08-17T17:31:37.156907+0000","last_clean":"2024-08-17T17:31:37.156907+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:31:37.156907+0000","last_undegraded":"2024-08-17T17:31:37.156907+0000","last_fullsized":"2024-08-17T17:31:37.156907+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":159,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.156313+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:31:36.156313+0000","last_peered":"2024-08-17T17:31:36.156313+0000","last_clean":"2024-08-17T17:31:36.156313+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:31:36.156313+0000","last_undegraded":"2024-08-17T17:31:36.156313+0000","last_fullsized":"2024-08-17T17:31:36.156313+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"249'6479","reported_seq":6532,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.547806+0000","last_change":"2024-08-17T17:29:57.200346+0000","last_active":"2024-08-17T17:31:48.547806+0000","last_peered":"2024-08-17T17:31:48.547806+0000","last_clean":"2024-08-17T17:31:48.547806+0000","last_became_active":"2024-08-17T17:29:57.200066+0000","last_became_peered":"2024-08-17T17:29:57.200066+0000","last_unstale":"2024-08-17T17:31:48.547806+0000","last_undegraded":"2024-08-17T17:31:48.547806+0000","last_fullsized":"2024-08-17T17:31:48.547806+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:39.347444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382140416,"num_objects":5831,"num_object_clones":0,"num_object_copies":17493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6479,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":39,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.156309+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:31:37.156309+0000","last_peered":"2024-08-17T17:31:37.156309+0000","last_clean":"2024-08-17T17:31:37.156309+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:31:37.156309+0000","last_undegraded":"2024-08-17T17:31:37.156309+0000","last_fullsized":"2024-08-17T17:31:37.156309+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":360,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153145+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:31:37.153145+0000","last_peered":"2024-08-17T17:31:37.153145+0000","last_clean":"2024-08-17T17:31:37.153145+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:31:37.153145+0000","last_undegraded":"2024-08-17T17:31:37.153145+0000","last_fullsized":"2024-08-17T17:31:37.153145+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"249'6536","reported_seq":6588,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.840035+0000","last_change":"2024-08-17T17:29:57.204892+0000","last_active":"2024-08-17T17:31:48.840035+0000","last_peered":"2024-08-17T17:31:48.840035+0000","last_clean":"2024-08-17T17:31:48.840035+0000","last_became_active":"2024-08-17T17:29:57.204591+0000","last_became_peered":"2024-08-17T17:29:57.204591+0000","last_unstale":"2024-08-17T17:31:48.840035+0000","last_undegraded":"2024-08-17T17:31:48.840035+0000","last_fullsized":"2024-08-17T17:31:48.840035+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:25.247583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382861312,"num_objects":5842,"num_object_clones":0,"num_object_copies":17526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6536,"num_write_kb":396096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":375,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150050+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:31:37.150050+0000","last_peered":"2024-08-17T17:31:37.150050+0000","last_clean":"2024-08-17T17:31:37.150050+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:31:37.150050+0000","last_undegraded":"2024-08-17T17:31:37.150050+0000","last_fullsized":"2024-08-17T17:31:37.150050+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.164417+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:31:36.164417+0000","last_peered":"2024-08-17T17:31:36.164417+0000","last_clean":"2024-08-17T17:31:36.164417+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:31:36.164417+0000","last_undegraded":"2024-08-17T17:31:36.164417+0000","last_fullsized":"2024-08-17T17:31:36.164417+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"249'6610","reported_seq":6705,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.553692+0000","last_change":"2024-08-17T17:31:17.661359+0000","last_active":"2024-08-17T17:31:48.553692+0000","last_peered":"2024-08-17T17:31:48.553692+0000","last_clean":"2024-08-17T17:31:48.553692+0000","last_became_active":"2024-08-17T17:31:17.599056+0000","last_became_peered":"2024-08-17T17:31:17.599056+0000","last_unstale":"2024-08-17T17:31:48.553692+0000","last_undegraded":"2024-08-17T17:31:48.553692+0000","last_fullsized":"2024-08-17T17:31:48.553692+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:52:19.118486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386400256,"num_objects":5896,"num_object_clones":0,"num_object_copies":17688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6610,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":165,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.160347+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:31:36.160347+0000","last_peered":"2024-08-17T17:31:36.160347+0000","last_clean":"2024-08-17T17:31:36.160347+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:31:36.160347+0000","last_undegraded":"2024-08-17T17:31:36.160347+0000","last_fullsized":"2024-08-17T17:31:36.160347+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.155965+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:31:36.155965+0000","last_peered":"2024-08-17T17:31:36.155965+0000","last_clean":"2024-08-17T17:31:36.155965+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:31:36.155965+0000","last_undegraded":"2024-08-17T17:31:36.155965+0000","last_fullsized":"2024-08-17T17:31:36.155965+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"249'6588","reported_seq":6642,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:49.647527+0000","last_change":"2024-08-17T17:31:37.408160+0000","last_active":"2024-08-17T17:31:49.647527+0000","last_peered":"2024-08-17T17:31:49.647527+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.395243+0000","last_became_peered":"2024-08-17T17:31:37.395243+0000","last_unstale":"2024-08-17T17:31:49.647527+0000","last_undegraded":"2024-08-17T17:31:37.192943+0000","last_fullsized":"2024-08-17T17:31:37.149514+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208280,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6588,"num_write_kb":395457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":-241}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.155739+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:31:36.155739+0000","last_peered":"2024-08-17T17:31:36.155739+0000","last_clean":"2024-08-17T17:31:36.155739+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:31:36.155739+0000","last_undegraded":"2024-08-17T17:31:36.155739+0000","last_fullsized":"2024-08-17T17:31:36.155739+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.162645+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:31:36.162645+0000","last_peered":"2024-08-17T17:31:36.162645+0000","last_clean":"2024-08-17T17:31:36.162645+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:31:36.162645+0000","last_undegraded":"2024-08-17T17:31:36.162645+0000","last_fullsized":"2024-08-17T17:31:36.162645+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"249'6502","reported_seq":6553,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:49.270554+0000","last_change":"2024-08-17T17:31:37.367363+0000","last_active":"2024-08-17T17:31:49.270554+0000","last_peered":"2024-08-17T17:31:49.270554+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.358516+0000","last_became_peered":"2024-08-17T17:31:37.358516+0000","last_unstale":"2024-08-17T17:31:49.270554+0000","last_undegraded":"2024-08-17T17:31:37.192795+0000","last_fullsized":"2024-08-17T17:31:37.150416+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379846656,"num_objects":5796,"num_object_clones":0,"num_object_copies":17388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6006,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":393536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-210}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":355,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150007+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:31:37.150007+0000","last_peered":"2024-08-17T17:31:37.150007+0000","last_clean":"2024-08-17T17:31:37.150007+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:31:37.150007+0000","last_undegraded":"2024-08-17T17:31:37.150007+0000","last_fullsized":"2024-08-17T17:31:37.150007+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":343,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.369188+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:31:37.369188+0000","last_peered":"2024-08-17T17:31:37.369188+0000","last_clean":"2024-08-17T17:31:37.369188+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:31:37.369188+0000","last_undegraded":"2024-08-17T17:31:37.369188+0000","last_fullsized":"2024-08-17T17:31:37.369188+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"249'6449","reported_seq":6501,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.850374+0000","last_change":"2024-08-17T17:29:57.204368+0000","last_active":"2024-08-17T17:31:48.850374+0000","last_peered":"2024-08-17T17:31:48.850374+0000","last_clean":"2024-08-17T17:31:48.850374+0000","last_became_active":"2024-08-17T17:29:57.204070+0000","last_became_peered":"2024-08-17T17:29:57.204070+0000","last_unstale":"2024-08-17T17:31:48.850374+0000","last_undegraded":"2024-08-17T17:31:48.850374+0000","last_fullsized":"2024-08-17T17:31:48.850374+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6449,"log_dups_size":0,"ondisk_log_size":6449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:00.628676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6449,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":560,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.158340+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:31:36.158340+0000","last_peered":"2024-08-17T17:31:36.158340+0000","last_clean":"2024-08-17T17:31:36.158340+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:31:36.158340+0000","last_undegraded":"2024-08-17T17:31:36.158340+0000","last_fullsized":"2024-08-17T17:31:36.158340+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":837,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.146433+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:31:36.146433+0000","last_peered":"2024-08-17T17:31:36.146433+0000","last_clean":"2024-08-17T17:31:36.146433+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:31:36.146433+0000","last_undegraded":"2024-08-17T17:31:36.146433+0000","last_fullsized":"2024-08-17T17:31:36.146433+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":339,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.162766+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:31:37.162766+0000","last_peered":"2024-08-17T17:31:37.162766+0000","last_clean":"2024-08-17T17:31:37.162766+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:31:37.162766+0000","last_undegraded":"2024-08-17T17:31:37.162766+0000","last_fullsized":"2024-08-17T17:31:37.162766+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153297+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:31:37.153297+0000","last_peered":"2024-08-17T17:31:37.153297+0000","last_clean":"2024-08-17T17:31:37.153297+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:31:37.153297+0000","last_undegraded":"2024-08-17T17:31:37.153297+0000","last_fullsized":"2024-08-17T17:31:37.153297+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"249'6387","reported_seq":6416,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:46.057865+0000","last_change":"2024-08-17T17:29:57.203033+0000","last_active":"2024-08-17T17:31:46.057865+0000","last_peered":"2024-08-17T17:31:46.057865+0000","last_clean":"2024-08-17T17:31:46.057865+0000","last_became_active":"2024-08-17T17:29:57.202766+0000","last_became_peered":"2024-08-17T17:29:57.202766+0000","last_unstale":"2024-08-17T17:31:46.057865+0000","last_undegraded":"2024-08-17T17:31:46.057865+0000","last_fullsized":"2024-08-17T17:31:46.057865+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:06.312698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390791168,"num_objects":5963,"num_object_clones":0,"num_object_copies":17889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":623,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.149719+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:31:36.149719+0000","last_peered":"2024-08-17T17:31:36.149719+0000","last_clean":"2024-08-17T17:31:36.149719+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:31:36.149719+0000","last_undegraded":"2024-08-17T17:31:36.149719+0000","last_fullsized":"2024-08-17T17:31:36.149719+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":360,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153321+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:31:37.153321+0000","last_peered":"2024-08-17T17:31:37.153321+0000","last_clean":"2024-08-17T17:31:37.153321+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:31:37.153321+0000","last_undegraded":"2024-08-17T17:31:37.153321+0000","last_fullsized":"2024-08-17T17:31:37.153321+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":357,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153594+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:31:37.153594+0000","last_peered":"2024-08-17T17:31:37.153594+0000","last_clean":"2024-08-17T17:31:37.153594+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:31:37.153594+0000","last_undegraded":"2024-08-17T17:31:37.153594+0000","last_fullsized":"2024-08-17T17:31:37.153594+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"249'6485","reported_seq":6537,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:48.903524+0000","last_change":"2024-08-17T17:29:57.202172+0000","last_active":"2024-08-17T17:31:48.903524+0000","last_peered":"2024-08-17T17:31:48.903524+0000","last_clean":"2024-08-17T17:31:48.903524+0000","last_became_active":"2024-08-17T17:29:57.201785+0000","last_became_peered":"2024-08-17T17:29:57.201785+0000","last_unstale":"2024-08-17T17:31:48.903524+0000","last_undegraded":"2024-08-17T17:31:48.903524+0000","last_fullsized":"2024-08-17T17:31:48.903524+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6485,"log_dups_size":0,"ondisk_log_size":6485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:28.967156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378994688,"num_objects":5783,"num_object_clones":0,"num_object_copies":17349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6485,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":758,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.368680+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:31:37.368680+0000","last_peered":"2024-08-17T17:31:37.368680+0000","last_clean":"2024-08-17T17:31:37.368680+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:31:37.368680+0000","last_undegraded":"2024-08-17T17:31:37.368680+0000","last_fullsized":"2024-08-17T17:31:37.368680+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":350,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149740+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:31:37.149740+0000","last_peered":"2024-08-17T17:31:37.149740+0000","last_clean":"2024-08-17T17:31:37.149740+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:31:37.149740+0000","last_undegraded":"2024-08-17T17:31:37.149740+0000","last_fullsized":"2024-08-17T17:31:37.149740+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":397,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150706+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:31:37.150706+0000","last_peered":"2024-08-17T17:31:37.150706+0000","last_clean":"2024-08-17T17:31:37.150706+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:31:37.150706+0000","last_undegraded":"2024-08-17T17:31:37.150706+0000","last_fullsized":"2024-08-17T17:31:37.150706+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"249'6554","reported_seq":6605,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:48.906377+0000","last_change":"2024-08-17T17:31:37.399805+0000","last_active":"2024-08-17T17:31:48.906377+0000","last_peered":"2024-08-17T17:31:48.906377+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.386849+0000","last_became_peered":"2024-08-17T17:31:37.386849+0000","last_unstale":"2024-08-17T17:31:48.906377+0000","last_undegraded":"2024-08-17T17:31:37.193155+0000","last_fullsized":"2024-08-17T17:31:37.153842+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381943808,"num_objects":5828,"num_object_clones":0,"num_object_copies":17484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6039,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-211}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":960,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.157529+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:31:37.157529+0000","last_peered":"2024-08-17T17:31:37.157529+0000","last_clean":"2024-08-17T17:31:37.157529+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:31:37.157529+0000","last_undegraded":"2024-08-17T17:31:37.157529+0000","last_fullsized":"2024-08-17T17:31:37.157529+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.368878+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:31:37.368878+0000","last_peered":"2024-08-17T17:31:37.368878+0000","last_clean":"2024-08-17T17:31:37.368878+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:31:37.368878+0000","last_undegraded":"2024-08-17T17:31:37.368878+0000","last_fullsized":"2024-08-17T17:31:37.368878+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151829+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:31:37.151829+0000","last_peered":"2024-08-17T17:31:37.151829+0000","last_clean":"2024-08-17T17:31:37.151829+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:31:37.151829+0000","last_undegraded":"2024-08-17T17:31:37.151829+0000","last_fullsized":"2024-08-17T17:31:37.151829+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"249'6339","reported_seq":6973,"reported_epoch":249,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:46.587314+0000","last_change":"2024-08-17T17:31:37.374397+0000","last_active":"2024-08-17T17:31:46.587314+0000","last_peered":"2024-08-17T17:31:46.587314+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.345454+0000","last_became_peered":"2024-08-17T17:31:37.345454+0000","last_unstale":"2024-08-17T17:31:46.587314+0000","last_undegraded":"2024-08-17T17:31:37.181806+0000","last_fullsized":"2024-08-17T17:31:37.151993+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:19.379468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384499712,"num_objects":5867,"num_object_clones":0,"num_object_copies":17601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":234}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":29,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.147168+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:31:36.147168+0000","last_peered":"2024-08-17T17:31:36.147168+0000","last_clean":"2024-08-17T17:31:36.147168+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:31:36.147168+0000","last_undegraded":"2024-08-17T17:31:36.147168+0000","last_fullsized":"2024-08-17T17:31:36.147168+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":642,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.152027+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:31:37.152027+0000","last_peered":"2024-08-17T17:31:37.152027+0000","last_clean":"2024-08-17T17:31:37.152027+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:31:37.152027+0000","last_undegraded":"2024-08-17T17:31:37.152027+0000","last_fullsized":"2024-08-17T17:31:37.152027+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":366,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151041+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:31:37.151041+0000","last_peered":"2024-08-17T17:31:37.151041+0000","last_clean":"2024-08-17T17:31:37.151041+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:31:37.151041+0000","last_undegraded":"2024-08-17T17:31:37.151041+0000","last_fullsized":"2024-08-17T17:31:37.151041+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"249'6402","reported_seq":6446,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:45.566410+0000","last_change":"2024-08-17T17:31:17.646602+0000","last_active":"2024-08-17T17:31:45.566410+0000","last_peered":"2024-08-17T17:31:45.566410+0000","last_clean":"2024-08-17T17:31:45.566410+0000","last_became_active":"2024-08-17T17:31:17.591931+0000","last_became_peered":"2024-08-17T17:31:17.591931+0000","last_unstale":"2024-08-17T17:31:45.566410+0000","last_undegraded":"2024-08-17T17:31:45.566410+0000","last_fullsized":"2024-08-17T17:31:45.566410+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:12:29.536081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394788864,"num_objects":6024,"num_object_clones":0,"num_object_copies":18072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6402,"num_write_kb":397632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":363,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151931+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:31:37.151931+0000","last_peered":"2024-08-17T17:31:37.151931+0000","last_clean":"2024-08-17T17:31:37.151931+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:31:37.151931+0000","last_undegraded":"2024-08-17T17:31:37.151931+0000","last_fullsized":"2024-08-17T17:31:37.151931+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":813,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151585+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:31:37.151585+0000","last_peered":"2024-08-17T17:31:37.151585+0000","last_clean":"2024-08-17T17:31:37.151585+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:31:37.151585+0000","last_undegraded":"2024-08-17T17:31:37.151585+0000","last_fullsized":"2024-08-17T17:31:37.151585+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":360,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153078+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:31:37.153078+0000","last_peered":"2024-08-17T17:31:37.153078+0000","last_clean":"2024-08-17T17:31:37.153078+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:31:37.153078+0000","last_undegraded":"2024-08-17T17:31:37.153078+0000","last_fullsized":"2024-08-17T17:31:37.153078+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"249'6528","reported_seq":6580,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.265556+0000","last_change":"2024-08-17T17:29:57.198724+0000","last_active":"2024-08-17T17:31:49.265556+0000","last_peered":"2024-08-17T17:31:49.265556+0000","last_clean":"2024-08-17T17:31:49.265556+0000","last_became_active":"2024-08-17T17:29:57.198432+0000","last_became_peered":"2024-08-17T17:29:57.198432+0000","last_unstale":"2024-08-17T17:31:49.265556+0000","last_undegraded":"2024-08-17T17:31:49.265556+0000","last_fullsized":"2024-08-17T17:31:49.265556+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:10.292012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376963072,"num_objects":5752,"num_object_clones":0,"num_object_copies":17256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6528,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":353,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153921+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:31:37.153921+0000","last_peered":"2024-08-17T17:31:37.153921+0000","last_clean":"2024-08-17T17:31:37.153921+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:31:37.153921+0000","last_undegraded":"2024-08-17T17:31:37.153921+0000","last_fullsized":"2024-08-17T17:31:37.153921+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":873,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151269+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:31:37.151269+0000","last_peered":"2024-08-17T17:31:37.151269+0000","last_clean":"2024-08-17T17:31:37.151269+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:31:37.151269+0000","last_undegraded":"2024-08-17T17:31:37.151269+0000","last_fullsized":"2024-08-17T17:31:37.151269+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":351,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149801+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:31:37.149801+0000","last_peered":"2024-08-17T17:31:37.149801+0000","last_clean":"2024-08-17T17:31:37.149801+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:31:37.149801+0000","last_undegraded":"2024-08-17T17:31:37.149801+0000","last_fullsized":"2024-08-17T17:31:37.149801+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"249'6549","reported_seq":6601,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.293075+0000","last_change":"2024-08-17T17:29:57.199611+0000","last_active":"2024-08-17T17:31:49.293075+0000","last_peered":"2024-08-17T17:31:49.293075+0000","last_clean":"2024-08-17T17:31:49.293075+0000","last_became_active":"2024-08-17T17:29:57.199174+0000","last_became_peered":"2024-08-17T17:29:57.199174+0000","last_unstale":"2024-08-17T17:31:49.293075+0000","last_undegraded":"2024-08-17T17:31:49.293075+0000","last_fullsized":"2024-08-17T17:31:49.293075+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:39.273888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380698624,"num_objects":5809,"num_object_clones":0,"num_object_copies":17427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6549,"num_write_kb":395456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":372,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153455+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:31:37.153455+0000","last_peered":"2024-08-17T17:31:37.153455+0000","last_clean":"2024-08-17T17:31:37.153455+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:31:37.153455+0000","last_undegraded":"2024-08-17T17:31:37.153455+0000","last_fullsized":"2024-08-17T17:31:37.153455+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":368,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.158444+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:31:36.158444+0000","last_peered":"2024-08-17T17:31:36.158444+0000","last_clean":"2024-08-17T17:31:36.158444+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:31:36.158444+0000","last_undegraded":"2024-08-17T17:31:36.158444+0000","last_fullsized":"2024-08-17T17:31:36.158444+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.186991+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:31:37.186991+0000","last_peered":"2024-08-17T17:31:37.186991+0000","last_clean":"2024-08-17T17:31:37.186991+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:31:37.186991+0000","last_undegraded":"2024-08-17T17:31:37.186991+0000","last_fullsized":"2024-08-17T17:31:37.186991+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153549+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:31:37.153549+0000","last_peered":"2024-08-17T17:31:37.153549+0000","last_clean":"2024-08-17T17:31:37.153549+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:31:37.153549+0000","last_undegraded":"2024-08-17T17:31:37.153549+0000","last_fullsized":"2024-08-17T17:31:37.153549+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":161,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.152655+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:31:37.152655+0000","last_peered":"2024-08-17T17:31:37.152655+0000","last_clean":"2024-08-17T17:31:37.152655+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:31:37.152655+0000","last_undegraded":"2024-08-17T17:31:37.152655+0000","last_fullsized":"2024-08-17T17:31:37.152655+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":349,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150928+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:31:37.150928+0000","last_peered":"2024-08-17T17:31:37.150928+0000","last_clean":"2024-08-17T17:31:37.150928+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:31:37.150928+0000","last_undegraded":"2024-08-17T17:31:37.150928+0000","last_fullsized":"2024-08-17T17:31:37.150928+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.152267+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:31:36.152267+0000","last_peered":"2024-08-17T17:31:36.152267+0000","last_clean":"2024-08-17T17:31:36.152267+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:31:36.152267+0000","last_undegraded":"2024-08-17T17:31:36.152267+0000","last_fullsized":"2024-08-17T17:31:36.152267+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.147216+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:31:36.147216+0000","last_peered":"2024-08-17T17:31:36.147216+0000","last_clean":"2024-08-17T17:31:36.147216+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:31:36.147216+0000","last_undegraded":"2024-08-17T17:31:36.147216+0000","last_fullsized":"2024-08-17T17:31:36.147216+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":316,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.151001+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:31:37.151001+0000","last_peered":"2024-08-17T17:31:37.151001+0000","last_clean":"2024-08-17T17:31:37.151001+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:31:37.151001+0000","last_undegraded":"2024-08-17T17:31:37.151001+0000","last_fullsized":"2024-08-17T17:31:37.151001+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":375,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153587+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:31:37.153587+0000","last_peered":"2024-08-17T17:31:37.153587+0000","last_clean":"2024-08-17T17:31:37.153587+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:31:37.153587+0000","last_undegraded":"2024-08-17T17:31:37.153587+0000","last_fullsized":"2024-08-17T17:31:37.153587+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":365,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149704+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:31:37.149704+0000","last_peered":"2024-08-17T17:31:37.149704+0000","last_clean":"2024-08-17T17:31:37.149704+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:31:37.149704+0000","last_undegraded":"2024-08-17T17:31:37.149704+0000","last_fullsized":"2024-08-17T17:31:37.149704+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":39,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.156885+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:31:37.156885+0000","last_peered":"2024-08-17T17:31:37.156885+0000","last_clean":"2024-08-17T17:31:37.156885+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:31:37.156885+0000","last_undegraded":"2024-08-17T17:31:37.156885+0000","last_fullsized":"2024-08-17T17:31:37.156885+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":370,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149909+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:31:37.149909+0000","last_peered":"2024-08-17T17:31:37.149909+0000","last_clean":"2024-08-17T17:31:37.149909+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:31:37.149909+0000","last_undegraded":"2024-08-17T17:31:37.149909+0000","last_fullsized":"2024-08-17T17:31:37.149909+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":347,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153591+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:31:37.153591+0000","last_peered":"2024-08-17T17:31:37.153591+0000","last_clean":"2024-08-17T17:31:37.153591+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:31:37.153591+0000","last_undegraded":"2024-08-17T17:31:37.153591+0000","last_fullsized":"2024-08-17T17:31:37.153591+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":351,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150278+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:31:37.150278+0000","last_peered":"2024-08-17T17:31:37.150278+0000","last_clean":"2024-08-17T17:31:37.150278+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:31:37.150278+0000","last_undegraded":"2024-08-17T17:31:37.150278+0000","last_fullsized":"2024-08-17T17:31:37.150278+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":367,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149909+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:31:37.149909+0000","last_peered":"2024-08-17T17:31:37.149909+0000","last_clean":"2024-08-17T17:31:37.149909+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:31:37.149909+0000","last_undegraded":"2024-08-17T17:31:37.149909+0000","last_fullsized":"2024-08-17T17:31:37.149909+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":135,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.154101+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:31:36.154101+0000","last_peered":"2024-08-17T17:31:36.154101+0000","last_clean":"2024-08-17T17:31:36.154101+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:31:36.154101+0000","last_undegraded":"2024-08-17T17:31:36.154101+0000","last_fullsized":"2024-08-17T17:31:36.154101+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":376,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.368956+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:31:37.368956+0000","last_peered":"2024-08-17T17:31:37.368956+0000","last_clean":"2024-08-17T17:31:37.368956+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:31:37.368956+0000","last_undegraded":"2024-08-17T17:31:37.368956+0000","last_fullsized":"2024-08-17T17:31:37.368956+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":352,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153030+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:31:37.153030+0000","last_peered":"2024-08-17T17:31:37.153030+0000","last_clean":"2024-08-17T17:31:37.153030+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:31:37.153030+0000","last_undegraded":"2024-08-17T17:31:37.153030+0000","last_fullsized":"2024-08-17T17:31:37.153030+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.154221+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:31:36.154221+0000","last_peered":"2024-08-17T17:31:36.154221+0000","last_clean":"2024-08-17T17:31:36.154221+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:31:36.154221+0000","last_undegraded":"2024-08-17T17:31:36.154221+0000","last_fullsized":"2024-08-17T17:31:36.154221+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":33,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.147427+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:31:36.147427+0000","last_peered":"2024-08-17T17:31:36.147427+0000","last_clean":"2024-08-17T17:31:36.147427+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:31:36.147427+0000","last_undegraded":"2024-08-17T17:31:36.147427+0000","last_fullsized":"2024-08-17T17:31:36.147427+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.153996+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:31:36.153996+0000","last_peered":"2024-08-17T17:31:36.153996+0000","last_clean":"2024-08-17T17:31:36.153996+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:31:36.153996+0000","last_undegraded":"2024-08-17T17:31:36.153996+0000","last_fullsized":"2024-08-17T17:31:36.153996+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":43,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.157756+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:31:37.157756+0000","last_peered":"2024-08-17T17:31:37.157756+0000","last_clean":"2024-08-17T17:31:37.157756+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:31:37.157756+0000","last_undegraded":"2024-08-17T17:31:37.157756+0000","last_fullsized":"2024-08-17T17:31:37.157756+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":372,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.149971+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:31:37.149971+0000","last_peered":"2024-08-17T17:31:37.149971+0000","last_clean":"2024-08-17T17:31:37.149971+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:31:37.149971+0000","last_undegraded":"2024-08-17T17:31:37.149971+0000","last_fullsized":"2024-08-17T17:31:37.149971+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":372,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.150304+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:31:37.150304+0000","last_peered":"2024-08-17T17:31:37.150304+0000","last_clean":"2024-08-17T17:31:37.150304+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:31:37.150304+0000","last_undegraded":"2024-08-17T17:31:37.150304+0000","last_fullsized":"2024-08-17T17:31:37.150304+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.161158+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:31:36.161158+0000","last_peered":"2024-08-17T17:31:36.161158+0000","last_clean":"2024-08-17T17:31:36.161158+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:31:36.161158+0000","last_undegraded":"2024-08-17T17:31:36.161158+0000","last_fullsized":"2024-08-17T17:31:36.161158+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":391,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.153446+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:31:37.153446+0000","last_peered":"2024-08-17T17:31:37.153446+0000","last_clean":"2024-08-17T17:31:37.153446+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:31:37.153446+0000","last_undegraded":"2024-08-17T17:31:37.153446+0000","last_fullsized":"2024-08-17T17:31:37.153446+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6124339224,"num_objects":93451,"num_object_clones":0,"num_object_copies":280353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93451,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104163,"num_write_kb":6323585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":337,"num_bytes_recovered":22085632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16571052032,"data_stored":16571039816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104163,"ondisk_log_size":104163,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201619,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2336644,"kb_used_data":1922428,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":91408508,"statfs":{"total":95995035648,"available":93602312192,"internally_reserved":0,"allocated":1968566272,"data_stored":1967181556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":424147403},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922116,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2245112,"kb_used_data":2171440,"kb_used_omap":7,"kb_used_meta":73656,"kb_avail":91500040,"statfs":{"total":95995035648,"available":93696040960,"internally_reserved":0,"allocated":2223554560,"data_stored":2222181122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":75424438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085639,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1607344,"kb_used_data":1556392,"kb_used_omap":7,"kb_used_meta":50936,"kb_avail":92137808,"statfs":{"total":95995035648,"available":94349115392,"internally_reserved":0,"allocated":1593745408,"data_stored":1592376327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":52158905},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986529,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2327228,"kb_used_data":1903156,"kb_used_omap":1,"kb_used_meta":424062,"kb_avail":91417924,"statfs":{"total":95995035648,"available":93611954176,"internally_reserved":0,"allocated":1948831744,"data_stored":1947454614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":434239942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435421,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3833192,"kb_used_data":3346016,"kb_used_omap":5,"kb_used_meta":487034,"kb_avail":89911960,"statfs":{"total":95995035648,"available":92069847040,"internally_reserved":0,"allocated":3426320384,"data_stored":3424958934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":498722889},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598947,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2779624,"kb_used_data":2252320,"kb_used_omap":7,"kb_used_meta":527288,"kb_avail":90965528,"statfs":{"total":95995035648,"available":93148700672,"internally_reserved":0,"allocated":2306375680,"data_stored":2305013579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":539943085},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762474,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2625252,"kb_used_data":2238172,"kb_used_omap":7,"kb_used_meta":387064,"kb_avail":91119900,"statfs":{"total":95995035648,"available":93306777600,"internally_reserved":0,"allocated":2291888128,"data_stored":2290529277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":396353953},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926000,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3475576,"kb_used_data":3002224,"kb_used_omap":7,"kb_used_meta":473144,"kb_avail":90269576,"statfs":{"total":95995035648,"available":92436045824,"internally_reserved":0,"allocated":3074277376,"data_stored":3072900897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":484500060},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3056603136,"data_stored":3056599064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2282164224,"data_stored":2282160152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2294677504,"data_stored":2294677504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3416522752,"data_stored":3416522752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1939996672,"data_stored":1939996672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1578041344,"data_stored":1578041344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":53874688,"data_stored":53870616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1949171712,"data_stored":1949171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:31:52.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_2 pgp_num 16 2024-08-17T17:31:52.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:52.359 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:52 smithi175 ceph-mon[101434]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:52.359 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:52 smithi175 ceph-mon[101434]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 700 op/s; 29754/279756 objects degraded (10.636%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:52.375 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:52.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:52.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:52.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:52.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:52 smithi064 ceph-mon[104516]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:52.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:52 smithi064 ceph-mon[104516]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 700 op/s; 29754/279756 objects degraded (10.636%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:52 smithi064 ceph-mon[109246]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:52 smithi064 ceph-mon[109246]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 8.9 MiB/s wr, 700 op/s; 29754/279756 objects degraded (10.636%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:31:52.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:52.658 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:52.762 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:52.797 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:52.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:52.938 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:52.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.014 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:53.014 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:53.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:53.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:53.174 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:53.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:53.303 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:53.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:53.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:53.466 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:53.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:53.572 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29754/279756 objects degraded (10.636%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:53.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:31:53.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29754/279756 objects degraded (10.636%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:53.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29754/279756 objects degraded (10.636%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:53.607 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:53.709 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:53.710 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:53.736 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:53.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:31:53.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:53.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.036 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:54.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:54.086 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:54.137 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:31:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:31:53] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-08-17T17:31:54.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:54.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:54.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:54.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:54.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:54.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:54 smithi064 ceph-mon[104516]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 716 op/s; 29598/278262 objects degraded (10.637%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:54.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:54 smithi064 ceph-mon[109246]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 716 op/s; 29598/278262 objects degraded (10.637%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:54.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:54.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:54.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:54 smithi175 ceph-mon[101434]: pgmap v777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 716 op/s; 29598/278262 objects degraded (10.637%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:54.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:54.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:54.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:54.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:54.995 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:55.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:55.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:55.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:55.248 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:55.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:55.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:55.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1460793569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1460793569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:55.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:55.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:55.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:55.604 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1460793569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:55.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:31:55.605 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:55.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:55.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:55.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.776 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:55.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:55.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:55.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:56.017 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:56.042 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:56.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:56.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:56.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:56.190 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:31:56.190 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:31:56.190 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.190 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:109815 7.5 7:ae39ffe1:::benchmark_data_smithi064_210574_object11006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-17T17:31:56.190 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:56.027036+0000", 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.16215876000000001, 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.16218550100000001, 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 109815 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:31:56.191 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027036+0000", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027036+0000", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027035+0000", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:31:56.192 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027043+0000", 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6049999999999998e-06 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027061+0000", 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7195000000000001e-05 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027068+0000", 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8359999999999992e-06 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027108+0000", 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9099e-05 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027205+0000", 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7411999999999995e-05 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:31:56.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.027285+0000", 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0339000000000004e-05 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.030302+0000", 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030164670000000001 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.031091+0000", 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00078873300000000001 2024-08-17T17:31:56.195 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:109820 7.5 7:aab99c9d:::benchmark_data_smithi064_210574_object11011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:56.036862+0000", 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.15233317299999999, 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.152412081, 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:31:56.196 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 109820 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036862+0000", 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.197 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036862+0000", 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036860+0000", 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:31:56.198 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036869+0000", 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5799999999999992e-06 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036881+0000", 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1712999999999999e-05 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036888+0000", 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2409999999999998e-06 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.036933+0000", 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5207999999999998e-05 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.037067+0000", 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013434800000000001 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.200 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.037195+0000", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000127778 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.040702+0000", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035072530000000001 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:56.040836+0000", 2024-08-17T17:31:56.201 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013330899999999999 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 2 2024-08-17T17:31:56.202 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:56.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:56.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:56.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:31:56.253 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:56.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:56.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:56.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:56.425 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:56.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:56.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:56.568 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:56.595 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:56 smithi175 ceph-mon[101434]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 KiB/s wr, 772 op/s; 29598/268914 objects degraded (11.006%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:56.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:56 smithi064 ceph-mon[104516]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 KiB/s wr, 772 op/s; 29598/268914 objects degraded (11.006%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:56.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:56 smithi064 ceph-mon[109246]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 KiB/s wr, 772 op/s; 29598/268914 objects degraded (11.006%); 1.9 MiB/s, 31 objects/s recovering 2024-08-17T17:31:56.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:56.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:56.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:56.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:56.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:57.002 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:31:57.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:57.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.118 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:57.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:57.219 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:57.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:57.354 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:31:57.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:57.412 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:57.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:57.555 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:57.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:57.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:57.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:57.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:57.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:57.981 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:57.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:58.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:58.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:58.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:58.399 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:58 smithi064 ceph-mon[104516]: pgmap v779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 29416/266907 objects degraded (11.021%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:31:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:31:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29416/266907 objects degraded (11.021%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:58 smithi064 ceph-mon[109246]: pgmap v779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 29416/266907 objects degraded (11.021%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:31:58.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:31:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29416/266907 objects degraded (11.021%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:58.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:58.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:58.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:58.644 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:58 smithi175 ceph-mon[101434]: pgmap v779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 29416/266907 objects degraded (11.021%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:31:58.644 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:31:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29416/266907 objects degraded (11.021%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.687 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:58.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:58.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:58.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:58.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:58.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:59.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:59.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:59.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:59.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:59.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:59.350 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:59.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:59.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:59.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:59.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:59.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:59.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:59.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:59.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:59.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:59.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:31:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:59.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:59.994 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:31:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:00.096 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:00.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:00.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:00.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:00.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:00.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:00 smithi064 ceph-mon[104516]: pgmap v780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29416/262050 objects degraded (11.225%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:00.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:00 smithi064 ceph-mon[109246]: pgmap v780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29416/262050 objects degraded (11.225%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:00.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:00.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:00.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:00 smithi175 ceph-mon[101434]: pgmap v780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29416/262050 objects degraded (11.225%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:00.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:00.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:00.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:00.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:01.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:01.113 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:01.215 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:01.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:01.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:01.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:01.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:01.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:01.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:01.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:01.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:01.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:01.834 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:32:01.834 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:32:01.834 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:32:01.834 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:32:01.834 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:32:01.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:01.909 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:02.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:02.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:02.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:02.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:02.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:02.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:02.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:02.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:02.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.623 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:02 smithi175 ceph-mon[101434]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 29416/257385 objects degraded (11.429%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:02 smithi064 ceph-mon[104516]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 29416/257385 objects degraded (11.429%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:02.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:02 smithi064 ceph-mon[109246]: pgmap v781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 29416/257385 objects degraded (11.429%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:02.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:02.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:02.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:02.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:02.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:02.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:03.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:03.140 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:03.166 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:03.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:03.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:03.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.489 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:03.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:03.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:03.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29416/257385 objects degraded (11.429%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:03.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29416/257385 objects degraded (11.429%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:03.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29416/257385 objects degraded (11.429%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:03.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:03.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:03.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:03.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:03.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:04.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:04.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:04.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:04.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:04.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:04.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:04.336 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:03] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:32:04.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:04.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 7 pgp_num to 16 2024-08-17T17:32:04.444 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:04.465 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:04.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:04.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:04 smithi175 ceph-mon[101434]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 29236/256107 objects degraded (11.416%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:04 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2075498321' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:04 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[104516]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 29236/256107 objects degraded (11.416%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2075498321' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[109246]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 29236/256107 objects degraded (11.416%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2075498321' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:04 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:32:04.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:04.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:04.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:04.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:04.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:05.017 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:05.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:05.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:05.160 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:05.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:05.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:05.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:05.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:05 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:32:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:05 smithi175 ceph-mon[101434]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:32:05.420 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:05.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:05.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:32:05.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:05.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:05 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:32:05.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:05 smithi064 ceph-mon[104516]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:32:05.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:05 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:32:05.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:05 smithi064 ceph-mon[109246]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:32:05.568 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:05.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:05.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:05.830 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:05.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:05.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:06.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:06.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:06.147 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:06.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:06.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:06.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:06.385 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:32:06.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:06.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:06.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:06.558 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:06.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:06 smithi175 ceph-mon[101434]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 29236/248424 objects degraded (11.769%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:32:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:06 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:06 smithi064 ceph-mon[104516]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 29236/248424 objects degraded (11.769%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:32:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:06 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:06 smithi064 ceph-mon[109246]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 29236/248424 objects degraded (11.769%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:32:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:06 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:06.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:06.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:06.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:06.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:06.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:07.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:07.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.140 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:07.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:07.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:07.279 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:07.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:07.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:07.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:07.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:07.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[104516]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:32:07.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:07.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:07.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:07 smithi064 ceph-mon[109246]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:32:07.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:07.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:07.632 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:07 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:07.632 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1748131163' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:07.632 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:07 smithi175 ceph-mon[101434]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:32:07.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:07.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:07.784 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:32:07.784 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:32:07.784 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:32:07.784 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:32:07.784 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72143 6.15 6:a9ad64ed:::benchmark_data_smithi064_185391_object72142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.43761199900001, 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.940724318, 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.785 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72143 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342051+0000", 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342050+0000", 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342077+0000", 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7270000000000001e-05 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342092+0000", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4783999999999999e-05 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342099+0000", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8870000000000003e-06 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342136+0000", 2024-08-17T17:32:07.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7243000000000002e-05 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.342143+0000", 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6950000000000003e-06 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.153747+0000", 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81160374499999999 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.789 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.153759+0000", 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1907e-05 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278060+0000", 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.124301014 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278182+0000", 2024-08-17T17:32:07.790 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012255999999999999 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.278289+0000", 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000106573 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281551+0000", 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032626669999999999 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.791 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281583+0000", 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1698e-05 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281591+0000", 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9920000000000007e-06 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.281628+0000", 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6358e-05 2024-08-17T17:32:07.792 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282776+0000", 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011481849999999999 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72144 6.15 6:ab663db7:::benchmark_data_smithi064_185391_object72143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.43284898600001, 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.941616505, 2024-08-17T17:32:07.793 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72144 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.794 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346814+0000", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346813+0000", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346869+0000", 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5569000000000003e-05 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.795 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346882+0000", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3366e-05 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346888+0000", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6810000000000003e-06 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346927+0000", 2024-08-17T17:32:07.796 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8804000000000003e-05 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.346935+0000", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0099999999999995e-06 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154103+0000", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.80716819699999998 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154109+0000", 2024-08-17T17:32:07.797 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0939999999999996e-06 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280421+0000", 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.126312218 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280512+0000", 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0737999999999998e-05 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.798 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.280608+0000", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6014999999999995e-05 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282931+0000", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023226420000000002 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282940+0000", 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6860000000000001e-06 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.799 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.287152+0000", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042121260000000001 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.287190+0000", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7676999999999999e-05 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288431+0000", 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001240903 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.800 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72163 6.15 6:afaee3c3:::benchmark_data_smithi064_185391_object72162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.37495302100001, 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.892213878, 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72163 2024-08-17T17:32:07.801 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404710+0000", 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.802 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404709+0000", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404752+0000", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3374000000000002e-05 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404764+0000", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2225000000000001e-05 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.803 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404771+0000", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6460000000000004e-06 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404813+0000", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2273e-05 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.404822+0000", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2819999999999996e-06 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.804 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154171+0000", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.74934978299999999 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154180+0000", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3820000000000004e-06 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.282979+0000", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1287988870000001 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.805 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.283134+0000", 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000155616 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.283301+0000", 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016718599999999999 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288488+0000", 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051863259999999998 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.806 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288494+0000", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3860000000000004e-06 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295291+0000", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0067970110000000004 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295355+0000", 2024-08-17T17:32:07.807 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4337000000000004e-05 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.296924+0000", 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0015688290000000001 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72199 6.15 6:ac10834f:::benchmark_data_smithi064_185391_object72198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.33044692999999, 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8490388900000001, 2024-08-17T17:32:07.808 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72199 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.809 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449216+0000", 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449215+0000", 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449260+0000", 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4409000000000001e-05 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.810 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449270+0000", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0675e-05 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449277+0000", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449335+0000", 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7503000000000002e-05 2024-08-17T17:32:07.811 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.812 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.812 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.812 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.449346+0000", 2024-08-17T17:32:07.812 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0767999999999999e-05 2024-08-17T17:32:07.812 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154204+0000", 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70485822300000001 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154209+0000", 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8110000000000002e-06 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.813 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284450+0000", 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.13024152 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284593+0000", 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142776 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.284753+0000", 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016029999999999999 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.814 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288500+0000", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003746494 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288532+0000", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2416999999999998e-05 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.297006+0000", 2024-08-17T17:32:07.815 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0084740590000000008 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.297060+0000", 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3687000000000003e-05 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298255+0000", 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011954139999999999 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.816 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72213 6.15 6:aa0bc0e7:::benchmark_data_smithi064_185391_object72212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.31730844499998, 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8406289010000001, 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72213 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462355+0000", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462354+0000", 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:07.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462421+0000", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7621999999999997e-05 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462434+0000", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2686e-05 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462439+0000", 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9690000000000002e-06 2024-08-17T17:32:07.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462479+0000", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9821999999999999e-05 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.462486+0000", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0090000000000001e-06 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154233+0000", 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.69174695500000005 2024-08-17T17:32:07.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154241+0000", 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9990000000000008e-06 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.285915+0000", 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1316743970000001 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.286033+0000", 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000117416 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.286175+0000", 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142476 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288562+0000", 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023867889999999998 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288567+0000", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8879999999999996e-06 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.301655+0000", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013088416 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.301712+0000", 2024-08-17T17:32:07.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6478999999999999e-05 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.302984+0000", 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001272118 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72242 6.15 6:aef0d0cc:::benchmark_data_smithi064_185391_object72241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.29331695600001, 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8174085250000001, 2024-08-17T17:32:07.824 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72242 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:32:07.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486346+0000", 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486458+0000", 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112156 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486465+0000", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.939e-06 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486469+0000", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2309999999999998e-06 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486501+0000", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1683999999999997e-05 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.486508+0000", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6470000000000001e-06 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154266+0000", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.66775841199999997 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154270+0000", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2599999999999999e-06 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288587+0000", 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1343165909999999 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288695+0000", 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010791 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.288799+0000", 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104202 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292495+0000", 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036959290000000001 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292533+0000", 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8092000000000001e-05 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303032+0000", 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01049899 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303069+0000", 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6548000000000003e-05 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303755+0000", 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068645100000000003 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.831 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72252 6.15 6:ac5ac986:::benchmark_data_smithi064_185391_object72251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.28390889500002, 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.808657811, 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72252 2024-08-17T17:32:07.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495754+0000", 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.833 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495753+0000", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495804+0000", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0812000000000001e-05 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495816+0000", 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1687e-05 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495842+0000", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6234e-05 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495878+0000", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6211999999999998e-05 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.495885+0000", 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4799999999999998e-06 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154296+0000", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.65841167099999998 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154303+0000", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.289677+0000", 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1353735899999999 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.289861+0000", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018380100000000001 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.290033+0000", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017293000000000001 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292541+0000", 2024-08-17T17:32:07.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025071519999999999 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292574+0000", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3439000000000003e-05 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303834+0000", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011259748 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303866+0000", 2024-08-17T17:32:07.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2712999999999997e-05 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304412+0000", 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054580899999999996 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72299 6.15 6:a8c7d24b:::benchmark_data_smithi064_185391_object72298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:32:07.839 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.23625954400001, 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.761700287, 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72299 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.840 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543404+0000", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543403+0000", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543462+0000", 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8947000000000003e-05 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543472+0000", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8120000000000002e-06 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543476+0000", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7079999999999996e-06 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543515+0000", 2024-08-17T17:32:07.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8852e-05 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.543523+0000", 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8779999999999998e-06 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154355+0000", 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.61083240000000005 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154363+0000", 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2389999999999996e-06 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291104+0000", 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1367416850000001 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291259+0000", 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000154337 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.291420+0000", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000160851 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295162+0000", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037425610000000002 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295222+0000", 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0024e-05 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304526+0000", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093043810000000005 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304577+0000", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0447e-05 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305104+0000", 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00052719400000000003 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.846 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72344 6.15 6:aacad454:::benchmark_data_smithi064_185391_object72343:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.19182453000002, 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.721979817, 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.847 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72344 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587839+0000", 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587838+0000", 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587951+0000", 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112562 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587960+0000", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.587966+0000", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9059999999999999e-06 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.588017+0000", 2024-08-17T17:32:07.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1319000000000001e-05 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.588027+0000", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6989999999999998e-06 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154387+0000", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56636031200000003 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154394+0000", 2024-08-17T17:32:07.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292667+0000", 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.138272902 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292811+0000", 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000143845 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.292995+0000", 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018369400000000001 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295212+0000", 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022173050000000001 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.853 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.295229+0000", 2024-08-17T17:32:07.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:07.855 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7161999999999999e-05 2024-08-17T17:32:07.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.308328+0000", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013098426 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.308377+0000", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9567999999999997e-05 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309819+0000", 2024-08-17T17:32:07.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014416019999999999 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72396 6.15 6:ab62b94c:::benchmark_data_smithi064_185391_object72395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.07536185700002, 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.609453421, 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.857 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72396 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704302+0000", 2024-08-17T17:32:07.858 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704301+0000", 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704360+0000", 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9194999999999998e-05 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.859 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704373+0000", 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2911999999999999e-05 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704380+0000", 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1980000000000004e-06 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704418+0000", 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8041999999999999e-05 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.704424+0000", 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.268e-06 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154429+0000", 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45000499599999999 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154438+0000", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0010000000000007e-06 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294019+0000", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.139580679 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294145+0000", 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012642600000000001 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.294242+0000", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6430999999999999e-05 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298309+0000", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040671079999999998 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298317+0000", 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1550000000000007e-06 2024-08-17T17:32:07.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.312477+0000", 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014159916 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.312535+0000", 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7658000000000003e-05 2024-08-17T17:32:07.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313755+0000", 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001220418 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72430 6.15 6:a946c8e2:::benchmark_data_smithi064_185391_object72429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:32:07.865 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 304.03850593200002, 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5734773990000002, 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72430 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:32:07.866 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741157+0000", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741156+0000", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741208+0000", 2024-08-17T17:32:07.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1790999999999999e-05 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741219+0000", 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1229999999999999e-05 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741224+0000", 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1599999999999997e-06 2024-08-17T17:32:07.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741267+0000", 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2917000000000003e-05 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.741274+0000", 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6120000000000004e-06 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154483+0000", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.41320958299999999 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154493+0000", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0259e-05 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298342+0000", 2024-08-17T17:32:07.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1438485890000001 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298501+0000", 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015924900000000001 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.298681+0000", 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000180046 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.303798+0000", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051164660000000001 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304429+0000", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00063155899999999996 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313814+0000", 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093843699999999995 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.313870+0000", 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6700000000000003e-05 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.314635+0000", 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00076452599999999996 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72541 6.15 6:a961463b:::benchmark_data_smithi064_185391_object72540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.873 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 303.90499096799999, 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4433222699999999, 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72541 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:32:07.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874672+0000", 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874671+0000", 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874754+0000", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2688999999999994e-05 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874767+0000", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3389e-05 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874772+0000", 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1989999999999997e-06 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.876 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874831+0000", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8782000000000001e-05 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:03.874843+0000", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1446e-05 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154528+0000", 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27968510499999999 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154539+0000", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1076e-05 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300346+0000", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1458074700000001 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300524+0000", 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017809 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.300662+0000", 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000137816 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304437+0000", 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003775011 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.304481+0000", 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3884000000000002e-05 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.316283+0000", 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011801936000000001 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.316359+0000", 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6062000000000005e-05 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.317995+0000", 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001635584 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72617 6.15 6:ab07db17:::benchmark_data_smithi064_185391_object72616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 303.72661674900002, 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2663090640000001, 2024-08-17T17:32:07.881 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72617 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053047+0000", 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053046+0000", 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053146+0000", 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010018600000000001 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053157+0000", 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0451e-05 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053162+0000", 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2109999999999997e-06 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053200+0000", 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8426000000000001e-05 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.053218+0000", 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7501999999999999e-05 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154575+0000", 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.101357234 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154584+0000", 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8710000000000003e-06 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305134+0000", 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1505501330000001 2024-08-17T17:32:07.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305234+0000", 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010040899999999999 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.305334+0000", 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9568999999999995e-05 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309844+0000", 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0045095530000000003 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309891+0000", 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7169000000000002e-05 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.318049+0000", 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0081580650000000008 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.318091+0000", 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2663000000000001e-05 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319356+0000", 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0012642319999999999 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.889 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72663 6.15 6:ae8c21f0:::benchmark_data_smithi064_185391_object72662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 303.65696500199999, 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.198120447, 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72663 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122698+0000", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122696+0000", 2024-08-17T17:32:07.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122724+0000", 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7628999999999999e-05 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122737+0000", 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.322e-05 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122744+0000", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1910000000000003e-06 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122804+0000", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9978000000000001e-05 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.122817+0000", 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.255e-05 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154621+0000", 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.031804333999999997 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154632+0000", 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0482999999999999e-05 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306247+0000", 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.151615823 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306371+0000", 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012342899999999999 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.306491+0000", 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012012 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309930+0000", 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0034388779999999998 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309938+0000", 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1159999999999999e-06 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.896 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319434+0000", 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0094958760000000003 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.319484+0000", 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0108999999999997e-05 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320819+0000", 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001334911 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.897 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35007.0:72666 6.15 6:afe642e1:::benchmark_data_smithi064_185391_object72665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 303.65186184499998, 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.194315598, 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35007", 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 72666 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.898 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127802+0000", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127793+0000", 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999919 2024-08-17T17:32:07.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127838+0000", 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4923999999999999e-05 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127853+0000", 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4499e-05 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.900 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127859+0000", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7619999999999998e-06 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127899+0000", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9755000000000001e-05 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.127906+0000", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9800000000000001e-06 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154668+0000", 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.026761565000000001 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:04.154676+0000", 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0090000000000007e-06 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307367+0000", 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.152691165 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.902 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307506+0000", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013873800000000001 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.307675+0000", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016890500000000001 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309944+0000", 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022697709999999999 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.903 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.309996+0000", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1415000000000002e-05 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320880+0000", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010883840000000001 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.320931+0000", 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1153999999999998e-05 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.904 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:27:05.322117+0000", 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0011863659999999999 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:114727 7.5 7:a3475a1e:::benchmark_data_smithi064_210574_object15918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:03.456039+0000", 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.3236243050000001, 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.039995714000000002, 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.905 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 114727 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456039+0000", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456039+0000", 2024-08-17T17:32:07.906 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456038+0000", 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456044+0000", 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8540000000000002e-06 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456055+0000", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1099e-05 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456061+0000", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7239999999999997e-06 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456101+0000", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0602000000000002e-05 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.908 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456194+0000", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3175000000000002e-05 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456273+0000", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8965e-05 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.462927+0000", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0066537289999999997 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.482017+0000", 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.019089722999999999 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.495958+0000", 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013941217000000001 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.495995+0000", 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6940000000000002e-05 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.496035+0000", 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9891000000000001e-05 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:114728 7.5 7:ac9ad5a1:::benchmark_data_smithi064_210574_object15919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:03.456068+0000", 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.3235952559999999, 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.040288727000000003, 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.911 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 114728 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456068+0000", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456068+0000", 2024-08-17T17:32:07.912 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456068+0000", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456070+0000", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.497e-06 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456076+0000", 2024-08-17T17:32:07.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.964e-06 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.456081+0000", 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5959999999999997e-06 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.459595+0000", 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035137950000000001 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.459740+0000", 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000145458 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.459887+0000", 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014659699999999999 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.464356+0000", 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0044695890000000004 2024-08-17T17:32:07.915 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.482944+0000", 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.018587293000000001 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.496272+0000", 2024-08-17T17:32:07.916 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013328599 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.496319+0000", 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7351000000000001e-05 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:03.496357+0000", 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7407000000000003e-05 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:115314 7.5 7:a27befb0:::benchmark_data_smithi064_210574_object16505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.373496+0000", 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.4061673959999998, 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.12660112500000001, 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 115314 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.918 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373496+0000", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373496+0000", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373495+0000", 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:07.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373501+0000", 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3180000000000003e-06 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373513+0000", 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1994000000000001e-05 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373519+0000", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.682e-06 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373558+0000", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8859000000000002e-05 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op must wait for map", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.373562+0000", 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2579999999999997e-06 2024-08-17T17:32:07.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.376183+0000", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026209470000000002 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493659+0000", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.117476329 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493788+0000", 2024-08-17T17:32:07.922 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012841999999999999 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.498599+0000", 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0048109930000000004 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.499881+0000", 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0012816749999999999 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:07.923 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.499973+0000", 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.234e-05 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.500054+0000", 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1527999999999996e-05 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.500097+0000", 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2746e-05 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115318 7.f e251/228 7:f423c493:::benchmark_data_smithi064_210574_object16509:head v 251'7252, mlcod=250'7250)", 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.386259+0000", 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.3934043329999999, 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.113166322, 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.386259+0000", 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.925 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.386259+0000", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.386259+0000", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967296 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.386262+0000", 2024-08-17T17:32:07.926 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3440000000000001e-06 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.495366+0000", 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.109103749 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.495375+0000", 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2320000000000007e-06 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.495418+0000", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3213000000000001e-05 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.495441+0000", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.283e-05 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.499392+0000", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0039506719999999997 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.499425+0000", 2024-08-17T17:32:07.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3439999999999998e-05 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115316 7.c e251/248 7:3b3fdb82:::benchmark_data_smithi064_210574_object16507:head v 251'7152, mlcod=0'0)", 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.391543+0000", 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.388120351, 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.106874463, 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:07.929 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391543+0000", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391543+0000", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391543+0000", 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:07.930 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391550+0000", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.824e-06 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493896+0000", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.102345623 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493904+0000", 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.994e-06 2024-08-17T17:32:07.931 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493981+0000", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7102999999999997e-05 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.494002+0000", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0737000000000002e-05 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.498367+0000", 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004365262 2024-08-17T17:32:07.932 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.498418+0000", 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0417000000000002e-05 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:07.933 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:32:07.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:08.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:08.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:08.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:08.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:08.349 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:08.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:08.403 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:08 smithi175 ceph-mon[101434]: pgmap v786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 29049/246732 objects degraded (11.774%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:08.403 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29049/246732 objects degraded (11.774%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:08.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:08.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:08.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:08 smithi064 ceph-mon[104516]: pgmap v786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 29049/246732 objects degraded (11.774%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:08.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29049/246732 objects degraded (11.774%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:08.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:08 smithi064 ceph-mon[109246]: pgmap v786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 29049/246732 objects degraded (11.774%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:08.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29049/246732 objects degraded (11.774%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.677 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:08.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:08.778 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:08.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:08.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:32:08.850 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:32:08.851 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:08.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:32:08.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:08.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:08.932 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:32:08.933 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:08.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:08.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:32:08.971 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:09.003 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:32:09.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:09.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:32:09.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:09.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:09.072 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:32:09.072 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:09.105 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:32:09.105 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:09.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:09.150 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:09.209 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:09.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:09.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:09.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:09.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:09.523 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:09.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:09.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:09.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:09.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:09.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/303169822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/303169822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/303169822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:09.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:09.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:09.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:09.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:09.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:32:10.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:10.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:10.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:10.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:10.255 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:32:10.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:32:10.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:10.381 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:10.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:10.416 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:10.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:10.531 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:10.633 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:10.662 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:10.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:10.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:10.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:10.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[104516]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 29049/245367 objects degraded (11.839%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:10.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:10.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:10.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[109246]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 29049/245367 objects degraded (11.839%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:10.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:10.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:10.889 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:10 smithi175 ceph-mon[101434]: pgmap v787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 29049/245367 objects degraded (11.839%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:32:10.889 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:32:10.889 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:10.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:10.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:11.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:11.053 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:11.053 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:11.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:11.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:11.223 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:11.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:11.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:11.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:11.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:11.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:11.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:11.723 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:11.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:11.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:32:11.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:11.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:11.921 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:11.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:12.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:12.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:12.105 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:12.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:12.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:12.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:12.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:12.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:12.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.646 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:12.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:12.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:12.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:12.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:12 smithi064 ceph-mon[104516]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 29049/237246 objects degraded (12.244%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:32:12.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:12 smithi064 ceph-mon[109246]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 29049/237246 objects degraded (12.244%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:32:12.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:12 smithi175 ceph-mon[101434]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 29049/237246 objects degraded (12.244%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:32:12.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:13.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.088 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:13.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:13.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:13.217 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:13.319 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:13.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:13.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:13.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:13.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:13.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:13.622 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:32:13.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 29049/237246 objects degraded (12.244%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:13.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:13.771 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:13.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:13.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 29049/237246 objects degraded (12.244%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:13.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:13.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 29049/237246 objects degraded (12.244%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:13.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:13.873 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:13.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:14.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.048 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:14.048 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:14.082 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:13] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.51.0" 2024-08-17T17:32:14.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:14.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:14.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:14.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:14.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:14.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:14.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:14.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:14.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[104516]: pgmap v789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28867/235890 objects degraded (12.237%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[104516]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[109246]: pgmap v789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28867/235890 objects degraded (12.237%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:14 smithi064 ceph-mon[109246]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:32:14.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:14.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:14.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:14.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:14.902 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:14 smithi175 ceph-mon[101434]: pgmap v789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28867/235890 objects degraded (12.237%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:14.902 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:14 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:14.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2047310301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:14.903 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:14 smithi175 ceph-mon[101434]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:32:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:14.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:32:15.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.047 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:15.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:15.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:15.216 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:15.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:15.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:15.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:15.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:15.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:15.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:15.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:15.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:15.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:15.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2238268702' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:15 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2238268702' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:15 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2238268702' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:15 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:15.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:16.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:16.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:16.251 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:16.273 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:16.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:16.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:16.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:16.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:16.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:16.666 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:32:16.696 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:16.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:16 smithi175 ceph-mon[101434]: pgmap v791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 28867/227205 objects degraded (12.705%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:16.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:16.788 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:16 smithi175 ceph-mon[101434]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:32:16.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:16.835 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:16.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:16.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:16.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[104516]: pgmap v791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 28867/227205 objects degraded (12.705%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:16.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:16.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[104516]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:32:16.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[109246]: pgmap v791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 28867/227205 objects degraded (12.705%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:32:16.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:16.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:16 smithi064 ceph-mon[109246]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:32:16.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:17.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:17.077 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:17.078 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:17.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:17.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:17.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:32:17.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:17.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:17.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:17.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:17.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.428 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:17.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:17.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:17.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:17.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:17.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:17.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:17.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:32:17.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:17.945 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:18.110 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:18.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:18.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:18.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:18.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:18.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:18.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:18.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:18.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:18.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:18.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:18.757 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:18.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:18.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:18.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:18 smithi175 ceph-mon[101434]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 28672/225327 objects degraded (12.725%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28672/225327 objects degraded (12.725%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:18.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:18 smithi064 ceph-mon[104516]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 28672/225327 objects degraded (12.725%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28672/225327 objects degraded (12.725%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:18 smithi064 ceph-mon[109246]: pgmap v793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 28672/225327 objects degraded (12.725%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28672/225327 objects degraded (12.725%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:19.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:19.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:32:19.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:19.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:19.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:19.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:19.218 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:19.229 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:19.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:19.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:19.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:19.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:19.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:19.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:19.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:19.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:19.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:19.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:19.986 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:19.986 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:19.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:20.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:20.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:20.173 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:20.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:20.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:20.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:20.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:20.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:20.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:20.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:20.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:20.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:20.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:20.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:20 smithi175 ceph-mon[101434]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 28672/224127 objects degraded (12.793%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:20.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:20 smithi064 ceph-mon[104516]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 28672/224127 objects degraded (12.793%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:20.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:20 smithi064 ceph-mon[109246]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 28672/224127 objects degraded (12.793%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:32:20.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:20.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:21.094 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:21.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:21.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:21.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:21.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:21.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:21.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:21.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:21.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:32:21.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:21.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:21.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:21.687 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:21.790 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:21.809 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:21.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.823 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:21.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:21.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:21.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:21.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:22.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:22.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:22.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:22.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:22.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:22.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:22.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:22.543 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:22.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:22.671 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:32:22.672 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:22.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:22.816 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:22.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[104516]: pgmap v795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 28672/216480 objects degraded (13.245%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3832300960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:22.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[109246]: pgmap v795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 28672/216480 objects degraded (13.245%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:22.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3832300960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:22.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:22.918 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:22.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:22.976 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:22 smithi175 ceph-mon[101434]: pgmap v795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 28672/216480 objects degraded (13.245%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:22.976 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:22 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3832300960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:22.976 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:23.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:23.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:23.186 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:23.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:23.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:23.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:23.444 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:32:23.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:23.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:23.483 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:32:23.484 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:23.500 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:23.535 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:32:23.535 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:23.570 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:32:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:23.602 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:23.603 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:32:23.604 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:23.642 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:32:23.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:23.678 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:32:23.678 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:23.691 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:32:23.692 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:32:23.692 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:32:23.692 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:32:23.692 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:32:23.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:32:23.715 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:23.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:23.768 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[104516]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28672/216480 objects degraded (13.245%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/89673972' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:23.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:23.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[109246]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:32:23.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28672/216480 objects degraded (13.245%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:23.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/89673972' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:23.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:23.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:23 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:23 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:23 smithi175 ceph-mon[101434]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:32:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28672/216480 objects degraded (13.245%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:23 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/89673972' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:24.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:24.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:24.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:24.146 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:23] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.51.0" 2024-08-17T17:32:24.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:24.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:24.313 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:24.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:24.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:32:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:24.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:24.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:24.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:24.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:24.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:24 smithi064 ceph-mon[104516]: pgmap v797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 28459/215226 objects degraded (13.223%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:32:24.726 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:24 smithi064 ceph-mon[109246]: pgmap v797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 28459/215226 objects degraded (13.223%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:32:24.734 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:24.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:24.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:24.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:24.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:24.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:25.016 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:24 smithi175 ceph-mon[101434]: pgmap v797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 28459/215226 objects degraded (13.223%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:32:25.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:25.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:25.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:25.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:25.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:25.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:25.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:25.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:25.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:25.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:25.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:25.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:25.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:25.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:25.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:25.873 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:25.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:25.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:26.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:26.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:26.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:26.548 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:26.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:26.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:26.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.727 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:32:26.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:32:26.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[104516]: pgmap v798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 28459/206502 objects degraded (13.781%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:26.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3610652249' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[109246]: pgmap v798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 28459/206502 objects degraded (13.781%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:26.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3610652249' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:26 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:26.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:26 smithi175 ceph-mon[101434]: pgmap v798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 28459/206502 objects degraded (13.781%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:26 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3610652249' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:26 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:32:26.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:26.999 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:27.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:27.221 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:32:27.221 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:32:27.221 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:32:27.221 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:32:27.221 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:98334 7.d e249/248 7:b6124e45:::benchmark_data_smithi064_210574_object98333:head v 249'6147, mlcod=0'0)", 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:40.146041+0000", 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 47.071922438999998, 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.057879358999999998, 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146041+0000", 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.222 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146041+0000", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146041+0000", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146079+0000", 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8285999999999999e-05 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.223 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146088+0000", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6119999999999999e-06 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146092+0000", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3229999999999999e-06 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146265+0000", 2024-08-17T17:32:27.224 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000172972 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146293+0000", 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7763000000000002e-05 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.203272+0000", 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.056979529000000001 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.225 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.203920+0000", 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000648136 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:98340 7.d e249/248 7:b5abd322:::benchmark_data_smithi064_210574_object98339:head v 249'6148, mlcod=0'0)", 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:40.146154+0000", 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 47.071809453, 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.057836361000000003, 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.226 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146154+0000", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146154+0000", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146154+0000", 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:27.227 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146185+0000", 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0923999999999997e-05 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146193+0000", 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3429999999999996e-06 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.228 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.146204+0000", 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0954e-05 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.157338+0000", 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011134562000000001 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.157369+0000", 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0796000000000001e-05 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.229 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.230 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.230 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.203926+0000", 2024-08-17T17:32:27.230 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.046556436999999999 2024-08-17T17:32:27.230 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.230 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.203990+0000", 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4596000000000003e-05 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:98678 7.1 e249/248 7:8c40592d:::benchmark_data_smithi064_210574_object98677:head v 249'6096, mlcod=233'81)", 2024-08-17T17:32:27.231 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:40.619922+0000", 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 46.598041019999997, 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.062943611999999996, 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619922+0000", 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.232 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619922+0000", 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619922+0000", 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619945+0000", 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2734999999999998e-05 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.233 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619952+0000", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9330000000000004e-06 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.619956+0000", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7450000000000003e-06 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.641224+0000", 2024-08-17T17:32:27.234 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.021267649 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.641248+0000", 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3774e-05 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.682812+0000", 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.041564479000000001 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.235 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:40.682866+0000", 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3603999999999997e-05 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:110883 7.1 e250/248 7:8e9c363b:::benchmark_data_smithi064_210574_object12074:head v 250'6841, mlcod=233'741)", 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:57.683582+0000", 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 29.534381048, 2024-08-17T17:32:27.236 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.063963958000000001, 2024-08-17T17:32:27.237 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.237 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.237 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.237 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.237 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683582+0000", 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683582+0000", 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.238 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683581+0000", 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683589+0000", 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8970000000000008e-06 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683605+0000", 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5965999999999999e-05 2024-08-17T17:32:27.239 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.683613+0000", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6790000000000005e-06 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.685837+0000", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022236909999999999 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.685865+0000", 2024-08-17T17:32:27.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7908000000000001e-05 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.747515+0000", 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.061650715000000002 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.747546+0000", 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1012000000000001e-05 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.241 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:110889 7.1 e250/248 7:8f2110b3:::benchmark_data_smithi064_210574_object12080:head v 250'6842, mlcod=233'741)", 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:57.688154+0000", 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 29.529809379, 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.059411643, 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.242 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688154+0000", 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688154+0000", 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688152+0000", 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.243 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688163+0000", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0625000000000001e-05 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688183+0000", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0528999999999999e-05 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.688192+0000", 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6419999999999996e-06 2024-08-17T17:32:27.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.693497+0000", 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0053051089999999997 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.693543+0000", 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6029000000000003e-05 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.747551+0000", 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.054008267999999998 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:57.747566+0000", 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4314999999999999e-05 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.246 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112111 7.d e250/248 7:b9940ed9:::benchmark_data_smithi064_210574_object13302:head v 250'7033, mlcod=0'0)", 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.939792923999999, 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.090243121999999995, 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.247 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278169+0000", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278175+0000", 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8599999999999998e-06 2024-08-17T17:32:27.248 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278183+0000", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0250000000000002e-06 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278189+0000", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2809999999999999e-06 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278223+0000", 2024-08-17T17:32:27.249 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4010000000000001e-05 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278241+0000", 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8352e-05 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368371+0000", 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.090129859000000007 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.250 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368414+0000", 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2823e-05 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112116 7.d e250/248 7:b5089113:::benchmark_data_smithi064_210574_object13307:head v 250'7034, mlcod=0'0)", 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.937020287999999, 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.087496358999999996, 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.251 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.252 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280942+0000", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280950+0000", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1940000000000001e-06 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280959+0000", 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9550000000000004e-06 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.253 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280965+0000", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5589999999999996e-06 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346103+0000", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.065137729000000005 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346129+0000", 2024-08-17T17:32:27.254 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6253000000000001e-05 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368417+0000", 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.022288006999999999 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368439+0000", 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2473e-05 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.255 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112122 7.c e250/248 7:3111c4d1:::benchmark_data_smithi064_210574_object13313:head v 250'6950, mlcod=0'0)", 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.933963998999999, 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.099167975000000005, 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:32:27.256 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283998+0000", 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.257 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.284009+0000", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0597e-05 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346054+0000", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.062044729 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346060+0000", 2024-08-17T17:32:27.258 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3489999999999997e-06 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346276+0000", 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000216386 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346300+0000", 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.355e-05 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.259 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.383117+0000", 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.036817132000000002 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.383167+0000", 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0305e-05 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.260 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112126 7.7 e250/248 7:ed0774d6:::benchmark_data_smithi064_210574_object13317:head v 250'7069, mlcod=0'0)", 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.287684+0000", 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.930279604999999, 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.066194877999999999, 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.287684+0000", 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.261 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.287684+0000", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.287683+0000", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.287691+0000", 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6940000000000001e-06 2024-08-17T17:32:27.262 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.345988+0000", 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058296863999999997 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346002+0000", 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3543999999999999e-05 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.263 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346603+0000", 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000601277 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346626+0000", 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3051999999999999e-05 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.353813+0000", 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.007187143 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.264 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.353879+0000", 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5341999999999999e-05 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112137 7.b e250/248 7:df044e52:::benchmark_data_smithi064_210574_object13328:head v 250'7073, mlcod=0'0)", 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.303148+0000", 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.91481563, 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.067373019000000006, 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.265 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.303148+0000", 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.303148+0000", 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.266 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.303147+0000", 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.303155+0000", 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5609999999999999e-06 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.267 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:27.268 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.268 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.345996+0000", 2024-08-17T17:32:27.268 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.042840912000000002 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346007+0000", 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1144e-05 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346767+0000", 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00075970499999999995 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.269 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346787+0000", 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0020000000000001e-05 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.370488+0000", 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.023700588000000002 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.370521+0000", 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3278000000000002e-05 2024-08-17T17:32:27.270 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:113900 7.b e250/248 7:d4934fe5:::benchmark_data_smithi064_210574_object15091:head v 250'7188, mlcod=0'0)", 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:02.221253+0000", 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 24.996710156999999, 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.070341564999999995, 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.271 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221253+0000", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221253+0000", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221252+0000", 2024-08-17T17:32:27.272 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221264+0000", 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2038000000000001e-05 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221279+0000", 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4909000000000001e-05 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.273 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221287+0000", 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2120000000000003e-06 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221331+0000", 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4162000000000002e-05 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.274 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.221355+0000", 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3515000000000001e-05 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.291553+0000", 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.070198700000000003 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.291595+0000", 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1581000000000001e-05 2024-08-17T17:32:27.275 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:113901 7.b e250/248 7:d67e9bdd:::benchmark_data_smithi064_210574_object15092:head v 250'7189, mlcod=0'0)", 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:02.224822+0000", 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 24.993141695999999, 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.066800614999999994, 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.276 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224822+0000", 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224822+0000", 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224821+0000", 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.277 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224838+0000", 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7592999999999999e-05 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224850+0000", 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1771e-05 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.278 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.224857+0000", 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0650000000000001e-06 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.229507+0000", 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0046498190000000003 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.229533+0000", 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6276999999999999e-05 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.279 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.291599+0000", 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.062065642999999997 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.291622+0000", 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3370000000000002e-05 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.280 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:113912 7.d e250/248 7:bedd2b7f:::benchmark_data_smithi064_210574_object15103:head v 250'7161, mlcod=0'0)", 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:02.233046+0000", 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 24.984917068000001, 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.064875756000000007, 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233046+0000", 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.281 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233046+0000", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233046+0000", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233053+0000", 2024-08-17T17:32:27.282 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6179999999999997e-06 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233063+0000", 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8360000000000004e-06 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233069+0000", 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5799999999999999e-06 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.283 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233109+0000", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0204999999999999e-05 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.233127+0000", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8600000000000001e-05 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.297791+0000", 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.064663878999999994 2024-08-17T17:32:27.284 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.297922+0000", 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000130875 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:113914 7.7 e250/248 7:e330c06d:::benchmark_data_smithi064_210574_object15105:head v 250'7184, mlcod=0'0)", 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:02.236807+0000", 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 24.981156889000001, 2024-08-17T17:32:27.285 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058794109999999997, 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.236807+0000", 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.236807+0000", 2024-08-17T17:32:27.286 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.236806+0000", 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.236813+0000", 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5889999999999997e-06 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.287 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.284607+0000", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.047794111 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.284622+0000", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.465e-05 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.284732+0000", 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000110389 2024-08-17T17:32:27.288 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.284760+0000", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7665e-05 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.295559+0000", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010799138 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.295601+0000", 2024-08-17T17:32:27.289 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1508000000000003e-05 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:113919 7.d e250/248 7:b9b0e9aa:::benchmark_data_smithi064_210574_object15110:head v 250'7162, mlcod=0'0)", 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:02.242404+0000", 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 24.975559041, 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069550445000000002, 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.290 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.242404+0000", 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.242404+0000", 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.291 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.242403+0000", 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.242412+0000", 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2400000000000007e-06 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.287865+0000", 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.045453759000000003 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.292 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.287877+0000", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1426999999999999e-05 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.289327+0000", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001450654 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.289352+0000", 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4482999999999999e-05 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.311909+0000", 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.022556924999999999 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:02.311955+0000", 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5921999999999998e-05 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.294 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115310 7.b e251/248 7:d3949251:::benchmark_data_smithi064_210574_object16501:head v 251'7274, mlcod=0'0)", 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 22.829437853000002, 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.118751128, 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:32:27.295 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388525+0000", 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.296 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388530+0000", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7790000000000003e-06 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388540+0000", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0487e-05 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388547+0000", 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5429999999999999e-06 2024-08-17T17:32:27.297 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388585+0000", 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8389000000000001e-05 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388601+0000", 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5571999999999999e-05 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.298 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.299 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507236+0000", 2024-08-17T17:32:27.299 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11863559999999999 2024-08-17T17:32:27.299 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507277+0000", 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0198999999999999e-05 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115316 7.c e251/248 7:3b3fdb82:::benchmark_data_smithi064_210574_object16507:head v 251'7152, mlcod=0'0)", 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 22.826398361999999, 2024-08-17T17:32:27.300 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11131099, 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:32:27.301 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391572+0000", 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1500000000000002e-06 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.302 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493796+0000", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.102224026 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493806+0000", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9459999999999993e-06 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493889+0000", 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3380000000000005e-05 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.303 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493916+0000", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6599000000000001e-05 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.502831+0000", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.008915002 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.502876+0000", 2024-08-17T17:32:27.304 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5386999999999998e-05 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115322 7.b e251/248 7:dc3829f1:::benchmark_data_smithi064_210574_object16513:head v 251'7275, mlcod=0'0)", 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 22.822046913000001, 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.113719247, 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.305 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:32:27.306 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395923+0000", 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2509999999999997e-06 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493591+0000", 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.097667976000000004 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.307 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493608+0000", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7342000000000001e-05 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.497919+0000", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043104470000000002 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.497943+0000", 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4584999999999999e-05 2024-08-17T17:32:27.308 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507284+0000", 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093404249999999994 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.509636+0000", 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023520199999999998 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.309 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:116602 7.7 e251/248 7:e91580cf:::benchmark_data_smithi064_210574_object17793:head v 251'7353, mlcod=0'0)", 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:06.261226+0000", 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 20.956737054000001, 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.068262970000000006, 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.310 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261226+0000", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261226+0000", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261224+0000", 2024-08-17T17:32:27.311 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261240+0000", 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5358000000000001e-05 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261271+0000", 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0864999999999997e-05 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.312 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.261282+0000", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1083999999999999e-05 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.271065+0000", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0097832400000000003 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.271113+0000", 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8143000000000003e-05 2024-08-17T17:32:27.313 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.329452+0000", 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.058339254 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.329489+0000", 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6977999999999997e-05 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.314 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:116606 7.7 e251/248 7:e5a29fb2:::benchmark_data_smithi064_210574_object17797:head v 251'7354, mlcod=0'0)", 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:06.265867+0000", 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 20.952095989, 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.063816566000000005, 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265867+0000", 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.315 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265867+0000", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265866+0000", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265878+0000", 2024-08-17T17:32:27.316 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1753000000000001e-05 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265897+0000", 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9032999999999999e-05 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.265906+0000", 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3000000000000007e-06 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:32:27.317 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.279668+0000", 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013762457000000001 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.279704+0000", 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5138000000000001e-05 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.329494+0000", 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.049790382000000001 2024-08-17T17:32:27.318 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:06.329684+0000", 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019006200000000001 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:32:27.319 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:32:27.333 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:27.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:27.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:27.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:32:27.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:27.702 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:27.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:27.731 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:27.833 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:27.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:27.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:32:27.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[104516]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:32:27.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[104516]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 28247/204507 objects degraded (13.812%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:27.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:32:27.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[109246]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:32:27.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:27 smithi064 ceph-mon[109246]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 28247/204507 objects degraded (13.812%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:27.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:28.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:28.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:28.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:28.093 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:27 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:32:28.093 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:27 smithi175 ceph-mon[101434]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:32:28.094 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:27 smithi175 ceph-mon[101434]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 28247/204507 objects degraded (13.812%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:28.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:28.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:32:28.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:28.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:28.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:28.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:28.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:28.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:28.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:28.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:28.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:28.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:28.784 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:28.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:28.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:28.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:28.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:29.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:29.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28247/204507 objects degraded (13.812%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28247/204507 objects degraded (13.812%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:29.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28247/204507 objects degraded (13.812%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:29.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.337 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.369 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:29.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:29.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:29.659 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:29.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:29.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:29.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:29.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:30.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:30.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:29 smithi064 ceph-mon[104516]: pgmap v801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 28247/203166 objects degraded (13.903%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:30.088 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:29 smithi064 ceph-mon[109246]: pgmap v801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 28247/203166 objects degraded (13.903%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.089 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:30.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:29 smithi175 ceph-mon[101434]: pgmap v801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 28247/203166 objects degraded (13.903%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:32:30.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:30.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:30.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:30.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:30.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:30.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:30.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:30.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:30.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:31.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:31.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:31.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:31.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:31.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:31.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:31.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:31.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:31.590 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:31.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:31.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:31.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:31.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:31.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:32.094 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:32.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:32 smithi064 ceph-mon[109246]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 28247/195321 objects degraded (14.462%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:32.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:32.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:32.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.371 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:32 smithi064 ceph-mon[104516]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 28247/195321 objects degraded (14.462%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:32.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:32 smithi175 ceph-mon[101434]: pgmap v802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 28247/195321 objects degraded (14.462%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:32.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.454 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:32:32.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:32.490 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:32:32.491 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:32.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:32:32.531 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:32.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:32.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:32.566 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:32:32.566 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:32.599 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:32:32.599 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:32.635 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:32:32.635 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:32.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.668 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:32:32.669 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:32.702 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:32:32.702 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:32.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:32.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:32.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:32.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:33.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:33.118 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:33.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:33.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:33.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:33.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:33.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2141364987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2141364987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:33.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2141364987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:33.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:33.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:33.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:33.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:33.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:33.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:33.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:33.804 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:33.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:33.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:34.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:34.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:34.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:34 smithi064 ceph-mon[104516]: pgmap v803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 28024/194082 objects degraded (14.439%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:34.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:34 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 28024/194082 objects degraded (14.439%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:34.153 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:33] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-08-17T17:32:34.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:34 smithi064 ceph-mon[109246]: pgmap v803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 28024/194082 objects degraded (14.439%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:34.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:34 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 28024/194082 objects degraded (14.439%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:34.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.214 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:34.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:34.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:34.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:34 smithi175 ceph-mon[101434]: pgmap v803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 28024/194082 objects degraded (14.439%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:34 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 28024/194082 objects degraded (14.439%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:34.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:34.482 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:34.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:34.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:34.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:34.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:34.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:34.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:34.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:35.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:35.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:35.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:35.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:35.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:35.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:35.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:35.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:35.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:35.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:35.695 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:35.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:35.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:35.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:35.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:35.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:36.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:36.118 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:36.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:36.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:36.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:36.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:36 smithi064 ceph-mon[104516]: pgmap v804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 28024/186045 objects degraded (15.063%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:36.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:36 smithi064 ceph-mon[109246]: pgmap v804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 28024/186045 objects degraded (15.063%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:36.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:36 smithi175 ceph-mon[101434]: pgmap v804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 28024/186045 objects degraded (15.063%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:32:36.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:36.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:36.568 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:36.597 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:36.597 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:36.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:36.716 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:36.818 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:36.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:36.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:36.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:37.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:37.124 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:37.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:37.257 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:37.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:37.664 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:37.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:37.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:37.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:38.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:38.092 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:38.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:38.228 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:38.330 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:38.362 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:38.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:38 smithi175 ceph-mon[101434]: pgmap v805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 27765/184374 objects degraded (15.059%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:32:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:38 smithi064 ceph-mon[104516]: pgmap v805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 27765/184374 objects degraded (15.059%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:32:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:38 smithi064 ceph-mon[109246]: pgmap v805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 27765/184374 objects degraded (15.059%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:32:38.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:38.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:38.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:38.642 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:38.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:38.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:38.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:38.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.930 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:38.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:39.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:39.075 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:39 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 27765/184374 objects degraded (15.059%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:39 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 27765/184374 objects degraded (15.059%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:39.222 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:39.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:39 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 27765/184374 objects degraded (15.059%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:32:39.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:39.515 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:39.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:39.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:39.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:39.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:39.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:39.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:40.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:40.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:32:40.160 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:32:40.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:40.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:40.247 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:40.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:40.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[104516]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 27765/183060 objects degraded (15.167%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/819408759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[109246]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 27765/183060 objects degraded (15.167%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/819408759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:40.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:40.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:40.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:40.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:40 smithi175 ceph-mon[101434]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 27765/183060 objects degraded (15.167%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:32:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/819408759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:40 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:40.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:40.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:40.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:40.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:41.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:41.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:41.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:41.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:41.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:41.340 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:41.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:41.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:41.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[104516]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:32:41.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:41.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:41.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:41 smithi064 ceph-mon[109246]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:32:41.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:41.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:41.576 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:41.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:41.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:41 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:41 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:41 smithi175 ceph-mon[101434]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:32:41.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:41.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:41.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:41.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:41.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:41.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:42.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:42.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:42.196 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:42.198 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:32:42.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:42.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:42.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[104516]: pgmap v808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 27765/175152 objects degraded (15.852%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:32:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/963869587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[109246]: pgmap v808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 27765/175152 objects degraded (15.852%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:32:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/963869587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:42 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:42.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:42.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.620 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:42.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:42 smithi175 ceph-mon[101434]: pgmap v808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 27765/175152 objects degraded (15.852%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:32:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/963869587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:42 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:32:42.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:42.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:42.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:42.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:43.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:43.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:43.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:43.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:43.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:43.355 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:43.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:32:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:43.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:43 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:32:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:43 smithi064 ceph-mon[104516]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:32:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:43 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:32:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:43 smithi064 ceph-mon[109246]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:32:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.539 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:43.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:43.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:43 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:32:43.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:43 smithi175 ceph-mon[101434]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:32:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:43.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:43.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:43.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:43.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:43.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:44.058 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:43] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:32:44.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:44.114 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:44.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:32:44.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:44.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:44.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:44.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:44 smithi064 ceph-mon[104516]: pgmap v810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 27497/173901 objects degraded (15.812%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:32:44.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:44 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 27497/173901 objects degraded (15.812%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:44 smithi064 ceph-mon[109246]: pgmap v810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 27497/173901 objects degraded (15.812%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:32:44.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:44 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 27497/173901 objects degraded (15.812%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:44.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.531 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:44.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:44.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:44 smithi175 ceph-mon[101434]: pgmap v810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 27497/173901 objects degraded (15.812%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:32:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:44 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 27497/173901 objects degraded (15.812%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:44.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:44.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:44.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:44.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:44.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:45.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:45.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:45.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:45.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:45.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:45.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:45.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:45.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:45.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:45.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:45.633 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:45.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:45.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:45.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:45.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:45.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:45.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:45.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:45.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:45.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:46.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:46.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:46.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:46.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:46.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:46.309 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:46.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:46.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:46 smithi064 ceph-mon[104516]: pgmap v811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 27497/165654 objects degraded (16.599%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:46.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1073386437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:46.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:46 smithi064 ceph-mon[109246]: pgmap v811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 27497/165654 objects degraded (16.599%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:46.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1073386437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:46.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:46.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:46.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:46.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:46 smithi175 ceph-mon[101434]: pgmap v811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 27497/165654 objects degraded (16.599%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1073386437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:46.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:46.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:46.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:46.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:46.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:47.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:47.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:47.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:47.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:47.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:47.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:47.488 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:47.527 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.552 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:47.654 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:47.679 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:47.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.811 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:47.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.940 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:47.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:48.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:48.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:48.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:48.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:48.214 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:32:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:32:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:32:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:32:48.215 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:32:48.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:48 smithi064 ceph-mon[104516]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 27216/163641 objects degraded (16.632%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:32:48.288 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:48 smithi064 ceph-mon[109246]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 27216/163641 objects degraded (16.632%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:32:48.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:48.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:48.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:48.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:48.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:48.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:48 smithi175 ceph-mon[101434]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 27216/163641 objects degraded (16.632%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:32:48.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:48.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:48.855 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:48.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:48.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:49.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:49.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:49.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:49.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:49.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:49.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:49.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:49.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:49.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:49.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:49 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 27216/163641 objects degraded (16.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:49.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:49 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 27216/163641 objects degraded (16.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:49.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:49.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.595 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:49.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:49 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 27216/163641 objects degraded (16.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:49.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:49.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:49.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:49.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:32:49.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:50.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:50.053 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:50.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:50.162 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:50.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:50.297 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:50.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:50.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.484 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:50.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:50.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:50 smithi064 ceph-mon[104516]: pgmap v813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 27216/162357 objects degraded (16.763%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:32:50.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:50 smithi064 ceph-mon[109246]: pgmap v813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 27216/162357 objects degraded (16.763%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:32:50.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:50.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:50 smithi175 ceph-mon[101434]: pgmap v813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 27216/162357 objects degraded (16.763%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:32:50.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.793 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:50.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:50.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:50.926 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:50.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:51.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:51.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:51.146 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:51.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:51.186 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:51.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:51.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:51.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:51.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:51.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:51.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:51.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:51.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.786 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:51.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:51.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:51.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:52.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:52.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:52.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:52.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:52.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:52.326 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:32:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:52.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:52.440 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:52.470 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:52.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:52.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:52 smithi175 ceph-mon[101434]: pgmap v814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 27216/154431 objects degraded (17.623%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:32:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3860684076' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:52 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[104516]: pgmap v814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 27216/154431 objects degraded (17.623%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:32:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3860684076' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[109246]: pgmap v814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 27216/154431 objects degraded (17.623%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:32:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3860684076' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:52 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:52.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:52.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:52.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:52.888 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:52.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:52.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:53.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:53.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:53.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:53.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:53.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:53.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:53.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:53.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:53.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:53.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[109246]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:32:53.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 27216/154431 objects degraded (17.623%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:53.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:53.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:53.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[104516]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:32:53.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 27216/154431 objects degraded (17.623%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:53.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:53 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:53 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:53 smithi175 ceph-mon[101434]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:32:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 27216/154431 objects degraded (17.623%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:53.684 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:53.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:53.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:53.945 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:53.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:54.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:54.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:32:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:32:53] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:32:54.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.260 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:54.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:54.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:54.544 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:54.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:54 smithi064 ceph-mon[104516]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26910/153114 objects degraded (17.575%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:54.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:54 smithi064 ceph-mon[109246]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26910/153114 objects degraded (17.575%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:54.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:54 smithi175 ceph-mon[101434]: pgmap v816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26910/153114 objects degraded (17.575%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:54.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:54.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:54.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.981 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:54.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:55.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:55.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:55.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:55.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:55.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:32:55.275 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:32:55.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:55.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:55.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/843976471' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/843976471' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:55.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:55.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:55.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/843976471' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.737 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:32:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.738 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:55.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:55.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.889 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:55.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:55.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:56.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:56.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:56.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:56.155 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:56.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:56.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:56.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:56.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:56.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:56.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:56 smithi175 ceph-mon[101434]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 26910/144657 objects degraded (18.603%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:56 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:32:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:56 smithi175 ceph-mon[101434]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:32:56.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:56.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[104516]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 26910/144657 objects degraded (18.603%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:32:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[104516]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:32:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[109246]: pgmap v817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 26910/144657 objects degraded (18.603%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:32:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:32:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:56 smithi064 ceph-mon[109246]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:32:56.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:56.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:56.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:56.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:56.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:57.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:57.118 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:57.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:57.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:57.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:57.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:57.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:57.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:57.538 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:57.640 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.684 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:57.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:57.786 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:57.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:57.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:58.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:58.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:58.190 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:58.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:58.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:58.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:58.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:58.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:58.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:58.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:58.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:58 smithi175 ceph-mon[101434]: pgmap v819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 26553/142725 objects degraded (18.604%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:32:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:32:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 26553/142725 objects degraded (18.604%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:58 smithi064 ceph-mon[104516]: pgmap v819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 26553/142725 objects degraded (18.604%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:32:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:32:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 26553/142725 objects degraded (18.604%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:58 smithi064 ceph-mon[109246]: pgmap v819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 26553/142725 objects degraded (18.604%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:32:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:32:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 26553/142725 objects degraded (18.604%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:58.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:58.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:58.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:58.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:59.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:59.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:59.306 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:59.333 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:59.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:32:59.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:59.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:32:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:59.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.654 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:59.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:59.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:32:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:00.032 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:00.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:00.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:00.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:00.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:00.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:00.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:00.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:00 smithi064 ceph-mon[104516]: pgmap v820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 26553/141489 objects degraded (18.767%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:33:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:00 smithi064 ceph-mon[109246]: pgmap v820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 26553/141489 objects degraded (18.767%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:33:00.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.472 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:00.552 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:00.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:00.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:00.626 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:33:00.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:00.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:00 smithi175 ceph-mon[101434]: pgmap v820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 26553/141489 objects degraded (18.767%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:33:00.668 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:33:00.669 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:00.699 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:00.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:33:00.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:00.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:00.749 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:33:00.750 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:00.785 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:33:00.785 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:00.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:33:00.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:00.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:00.853 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:00.856 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:33:00.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:00.891 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:33:00.891 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:00.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.983 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:01.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:01.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:01.229 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:01.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:01.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:01.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:01.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:01.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:01.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2584376372' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:01.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2584376372' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:01.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:01.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2584376372' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.672 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:01.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:01.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:01.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:01.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:01.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:01.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:02.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.085 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:02.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:02.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:02.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:02.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:02.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:02 smithi064 ceph-mon[104516]: pgmap v821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 26553/134076 objects degraded (19.804%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:02.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:02 smithi064 ceph-mon[109246]: pgmap v821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 26553/134076 objects degraded (19.804%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:02.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:02.509 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:02.611 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:02.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:02.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:02.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:02 smithi175 ceph-mon[101434]: pgmap v821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 26553/134076 objects degraded (19.804%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:02.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:02.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:02.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:02.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:02.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:02.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:03.016 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:03.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:03.154 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:03.256 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:03.267 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:03.267 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:03.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:03.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:03.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:03.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 26553/134076 objects degraded (19.804%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:03.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 26553/134076 objects degraded (19.804%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:03.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 26553/134076 objects degraded (19.804%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:03.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:03.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:03.796 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:03.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:03.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:03.954 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:04.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:04.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:04.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:04.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:04.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:04.326 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:03] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.51.0" 2024-08-17T17:33:04.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:04.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:04.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:04.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:04.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:04 smithi064 ceph-mon[104516]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 26215/132765 objects degraded (19.745%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:04.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:04 smithi064 ceph-mon[109246]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 26215/132765 objects degraded (19.745%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:04.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:04 smithi175 ceph-mon[101434]: pgmap v822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 26215/132765 objects degraded (19.745%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:04.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:04.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:04.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:04.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:05.208 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:05.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:05.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.464 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:05.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:05.593 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.619 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:05.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:05.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:05.748 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:05.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:33:05.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:05.874 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:05.976 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:06.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:06.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:06.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.159 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:06.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:06.260 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:06.285 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:06.353 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:33:06.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[104516]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 26215/125016 objects degraded (20.969%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:06.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3355923266' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[109246]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 26215/125016 objects degraded (20.969%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:06.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3355923266' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:06 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:06.434 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:06.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:06.564 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:06 smithi175 ceph-mon[101434]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 26215/125016 objects degraded (20.969%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:33:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:06 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3355923266' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:06 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:06.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:06.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:06.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:06.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:06.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:06.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:07.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:07.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:07.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:07.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:07.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:07.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:07.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:07.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:07.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[104516]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:33:07.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:07.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:07.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:07 smithi064 ceph-mon[109246]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:33:07.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:07.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:07.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:07.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:07 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:07 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:07.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:07 smithi175 ceph-mon[101434]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:33:07.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:07.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:07.800 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:07.937 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.039 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:08.068 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:08.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:08.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:08.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:08.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:08.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:08.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:08.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:08.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:08 smithi175 ceph-mon[101434]: pgmap v825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/123264 objects degraded (20.918%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25784/123264 objects degraded (20.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:08.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:08 smithi064 ceph-mon[104516]: pgmap v825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/123264 objects degraded (20.918%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25784/123264 objects degraded (20.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:08 smithi064 ceph-mon[109246]: pgmap v825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/123264 objects degraded (20.918%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25784/123264 objects degraded (20.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:08.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:08.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:08.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:08.930 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:09.031 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:09.032 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:09.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:09.084 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:09.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:09.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:09.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:09.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:09.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:09.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:09.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:09.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:09.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:09.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:09.748 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:09.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:09.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:09.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:10.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:10.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:10.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:10.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:10.326 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:10.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:33:10.464 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:10.488 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:10.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[104516]: pgmap v826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/122022 objects degraded (21.131%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:10.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:10.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:10.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[109246]: pgmap v826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/122022 objects degraded (21.131%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:10.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:10.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:10.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:10.614 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:10 smithi175 ceph-mon[101434]: pgmap v826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 25784/122022 objects degraded (21.131%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:10.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:10.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:10.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:10.874 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:10.976 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:11.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:11.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:11.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:11.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:11.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:11.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:11.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:11.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:11.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:11.669 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:11.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:11.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:11.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:11.945 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:12.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:12.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:12.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:12.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:12.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:12.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:12.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:12.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:12.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:12.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:12 smithi175 ceph-mon[101434]: pgmap v827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 25784/114510 objects degraded (22.517%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:12.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:12 smithi064 ceph-mon[104516]: pgmap v827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 25784/114510 objects degraded (22.517%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:12 smithi064 ceph-mon[109246]: pgmap v827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 25784/114510 objects degraded (22.517%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:33:12.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:12.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:12.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:13.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:13.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:13.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:13.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:13.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:13.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:13.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:13.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:13.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:13.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 25784/114510 objects degraded (22.517%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:13.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 25784/114510 objects degraded (22.517%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:13.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 25784/114510 objects degraded (22.517%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:13.700 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:13.802 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:13.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:13.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:13.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:14.071 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:13] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:33:14.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:14.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:14.144 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:14.144 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:14.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:14.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:14.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:14.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:33:14.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:14.532 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:14.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:14 smithi175 ceph-mon[101434]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25338/113172 objects degraded (22.389%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:33:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:33:14.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:14 smithi064 ceph-mon[104516]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25338/113172 objects degraded (22.389%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:33:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:33:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:14 smithi064 ceph-mon[109246]: pgmap v828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 25338/113172 objects degraded (22.389%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:33:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:33:14.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:14.813 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:14.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:14.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:14.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:15.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:15.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:15.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:33:15.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:15.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:15.369 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:15.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:15.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:33:15.439 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:15.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:15.488 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:33:15.488 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:15.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:15.536 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:33:15.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:15.575 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:33:15.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:15.589 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:15.616 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:33:15.616 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:15.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:15.648 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:15.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:33:15.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:33:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[104516]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:33:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3459561594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:15.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:33:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[109246]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:33:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3459561594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3015067903' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:33:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:15 smithi175 ceph-mon[101434]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:33:15.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3459561594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:15.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:33:15.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:15.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:33:15.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:15.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:15.793 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:15.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:16.032 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:16.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:16.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:33:16.158 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:16.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:16.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:16.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:16.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:16.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:16.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:16.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:16.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:16 smithi175 ceph-mon[101434]: pgmap v830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 25338/105129 objects degraded (24.102%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:33:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:16 smithi064 ceph-mon[104516]: pgmap v830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 25338/105129 objects degraded (24.102%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:33:16.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:16 smithi064 ceph-mon[109246]: pgmap v830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 25338/105129 objects degraded (24.102%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:33:16.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.777 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:16.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:16.878 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:16.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:17.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.079 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:17.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:17.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:17.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:17.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:17.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:17.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:17.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:17.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:17.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:17.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:17.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:17.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:17.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:17.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:18.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:18.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:18.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:18.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:18.313 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:18.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:18.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:18.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:18.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:18 smithi064 ceph-mon[104516]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/103254 objects degraded (24.017%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:18.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 24799/103254 objects degraded (24.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:18.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:18 smithi064 ceph-mon[109246]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/103254 objects degraded (24.017%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:18.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 24799/103254 objects degraded (24.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:18.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:18.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:18.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:18.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:18.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:18.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:18 smithi175 ceph-mon[101434]: pgmap v831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/103254 objects degraded (24.017%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 24799/103254 objects degraded (24.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:19.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:19.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:19.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:19.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:19.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:19.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:19.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:19.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:19.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:19.723 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:19.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:19.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:19.989 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:20.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:20.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:20.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:20.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:20.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:20.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:20.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.492 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.600 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:20.619 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:20.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.761 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:20.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:20.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:20 smithi064 ceph-mon[104516]: pgmap v832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/102030 objects degraded (24.306%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:20.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:20 smithi064 ceph-mon[109246]: pgmap v832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/102030 objects degraded (24.306%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:20.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.890 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:20 smithi175 ceph-mon[101434]: pgmap v832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24799/102030 objects degraded (24.306%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:20.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:20.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:21.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:21.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:21.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:21.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:21.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:21.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:21.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:21.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:21.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:21.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:21.672 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:33:21.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:21.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:21.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:21.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/338199351' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:21.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/338199351' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:21.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.004 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:22.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/338199351' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.254 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.480 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:22.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:22.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:22.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:22.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[104516]: pgmap v833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24799/94155 objects degraded (26.338%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[104516]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[109246]: pgmap v833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24799/94155 objects degraded (26.338%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:22.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:22 smithi064 ceph-mon[109246]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:33:23.031 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:23.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:23.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:22 smithi175 ceph-mon[101434]: pgmap v833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24799/94155 objects degraded (26.338%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:33:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:22 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:22 smithi175 ceph-mon[101434]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:33:23.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:23.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:23.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:23.427 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:23.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:23.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:23.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:23.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:23.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:23.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:23.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:23.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:23 smithi064 ceph-mon[104516]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:33:23.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 24799/94155 objects degraded (26.338%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:23.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:23 smithi064 ceph-mon[109246]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:33:23.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 24799/94155 objects degraded (26.338%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:23.989 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:24.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:24.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:23 smithi175 ceph-mon[101434]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:33:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 24799/94155 objects degraded (26.338%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:24.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:24.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:24.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.352 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:23] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:33:24.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:24.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:24.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:24.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:24.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:24.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:24 smithi064 ceph-mon[109246]: pgmap v836: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 24121/92844 objects degraded (25.980%); 8.1 MiB/s, 152 objects/s recovering 2024-08-17T17:33:24.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:24 smithi064 ceph-mon[104516]: pgmap v836: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 24121/92844 objects degraded (25.980%); 8.1 MiB/s, 152 objects/s recovering 2024-08-17T17:33:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:24.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:25.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:25.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:24 smithi175 ceph-mon[101434]: pgmap v836: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 24121/92844 objects degraded (25.980%); 8.1 MiB/s, 152 objects/s recovering 2024-08-17T17:33:25.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:25.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:25.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:25.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:25.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:25.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:25.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:25.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:25.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:25.809 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:25.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:25.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:25.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:25.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:25.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:26.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:26.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:26.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:26.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:26.193 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:33:26.193 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:33:26.193 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:33:26.193 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:33:26.193 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:33:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:26.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:26.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:26.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:26.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:26.457 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:26.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:26.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:26.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:26.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:26.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:26.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:26.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:33:26.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:26.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:27.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:27.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:26 smithi175 ceph-mon[101434]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24005/87087 objects degraded (27.564%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:27.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:26 smithi064 ceph-mon[104516]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24005/87087 objects degraded (27.564%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:27.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:26 smithi064 ceph-mon[109246]: pgmap v837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 24005/87087 objects degraded (27.564%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:27.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:27.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.347 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:27.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:27.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:27.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:27.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:27.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:27.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:27.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:27.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:27.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:28.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:28.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:28.244 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.413 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:28.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:28.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:28.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:28.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:28.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:28.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:28.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:28.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:29.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:29.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:28 smithi175 ceph-mon[101434]: pgmap v838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 24005/84234 objects degraded (28.498%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:28 smithi064 ceph-mon[104516]: pgmap v838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 24005/84234 objects degraded (28.498%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:28 smithi064 ceph-mon[109246]: pgmap v838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 24005/84234 objects degraded (28.498%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:29.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:29.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:29.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.561 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:29.839 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:29.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:30.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:30.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:30.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:30.217 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:30.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.357 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:30.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:30.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:30.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:30.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:30.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:33:30.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:30.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:30.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:30.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:30.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:30 smithi064 ceph-mon[104516]: pgmap v839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:30.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:30 smithi064 ceph-mon[109246]: pgmap v839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:30.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:30.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:31.003 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:33:31.004 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:31.035 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:33:31.035 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:31.045 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:31.068 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:31.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:31.072 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:33:31.073 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:31.111 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:33:31.112 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:31.147 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:33:31.148 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:31.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:30 smithi175 ceph-mon[101434]: pgmap v839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 24005/84234 objects degraded (28.498%); 4.8 MiB/s, 99 objects/s recovering 2024-08-17T17:33:31.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:31.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:33:31.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:31.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:31.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:33:31.221 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:31.260 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:33:31.260 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:31.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:31.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:31.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:31.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:31.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:31.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1033734869' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:31.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1033734869' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:31.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:31.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:31.979 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:32.004 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:32.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:32.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:32.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1033734869' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:32.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:32.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.280 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:32.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:32.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:32.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:32.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:32.546 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:32.648 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:32.674 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:32.748 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:33:32.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:32.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:32.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:32.965 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:33.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:33.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:32 smithi175 ceph-mon[101434]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23888/76209 objects degraded (31.345%); 4.8 MiB/s, 96 objects/s recovering 2024-08-17T17:33:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4175242344' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[104516]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23888/76209 objects degraded (31.345%); 4.8 MiB/s, 96 objects/s recovering 2024-08-17T17:33:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4175242344' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[109246]: pgmap v840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23888/76209 objects degraded (31.345%); 4.8 MiB/s, 96 objects/s recovering 2024-08-17T17:33:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4175242344' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:33:33.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:33.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:33.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:33.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:33.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:33.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:33.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:33.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:33.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:33.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:33.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:33.921 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:34.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:34.063 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:34.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:33:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:33 smithi175 ceph-mon[101434]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:33:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23888/76209 objects degraded (31.345%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:34.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:34.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:33:34.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[104516]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:33:34.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23888/76209 objects degraded (31.345%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:34.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:33] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-08-17T17:33:34.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:33:34.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[109246]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:33:34.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23888/76209 objects degraded (31.345%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:34.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:34.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:34.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:34.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:34.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:34.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:34.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:34.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:34.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:34.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:34.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:34.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:34.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:35.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:35.011 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:35.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:34 smithi175 ceph-mon[101434]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23888/75102 objects degraded (31.807%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:33:35.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:34 smithi064 ceph-mon[104516]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23888/75102 objects degraded (31.807%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:33:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:34 smithi064 ceph-mon[109246]: pgmap v842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23888/75102 objects degraded (31.807%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:33:35.203 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:35.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:35.305 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:35.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:35.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.469 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:35.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:35.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:35.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:35.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:35.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:35.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:35.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:35.862 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:35.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:35.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:33:35.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:36.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:36.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:35 smithi175 ceph-mon[101434]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 23760/69090 objects degraded (34.390%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:36.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:35 smithi064 ceph-mon[104516]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 23760/69090 objects degraded (34.390%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:36.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:35 smithi064 ceph-mon[109246]: pgmap v843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 23760/69090 objects degraded (34.390%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:36.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:36.279 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:36.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:36.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:36.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:36.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:36.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:36.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:36.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:36.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:36.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:36.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:36.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:36.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:37.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:37.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:37.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:37.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:37.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:37.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:37.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:37.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:37.530 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:37.632 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:37.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:37.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:37.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:37.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:37.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:38.039 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:38.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:38.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:38.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:38.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:38.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:38.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:38 smithi175 ceph-mon[101434]: pgmap v844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:38 smithi064 ceph-mon[104516]: pgmap v844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:38 smithi064 ceph-mon[109246]: pgmap v844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:38.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:38.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:38.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:38.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:38.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:38.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:38.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:38.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:38.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:39.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:39.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:39.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:39.260 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:39.284 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:39.386 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:39.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:39 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23760/66180 objects degraded (35.902%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:39.412 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:39.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:39 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23760/66180 objects degraded (35.902%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:39 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23760/66180 objects degraded (35.902%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:39.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:39.536 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:39.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:39.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:39.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:39.800 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:39.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:39.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:40.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:40.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:40.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:40.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:40.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:40 smithi175 ceph-mon[101434]: pgmap v845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:40.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:40 smithi064 ceph-mon[104516]: pgmap v845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:40 smithi064 ceph-mon[109246]: pgmap v845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23760/66180 objects degraded (35.902%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:40.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:40.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:40.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:40.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:40.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:40.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:41.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:41.051 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:41.099 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:41.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:41.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:41.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:41.352 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:41.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.482 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:41.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:41.518 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:41.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:41.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:41.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:41.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:41.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:41.967 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:42.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:42.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:42 smithi064 ceph-mon[104516]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23634/58083 objects degraded (40.690%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:42.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:42 smithi064 ceph-mon[109246]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23634/58083 objects degraded (40.690%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:42.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:42.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:42.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:42.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:42.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:42.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:42 smithi175 ceph-mon[101434]: pgmap v846: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 23634/58083 objects degraded (40.690%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:42.492 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:42.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:42.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:42.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:42.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:42.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:42.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:42.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:43.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:43.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:43.173 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:33:43.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:43 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/735782683' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:43 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:43 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/735782683' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:43 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:43.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:43.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:43.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:43 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/735782683' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:43 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:43.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:43.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:43.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:43.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:43.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:43.714 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:43.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:43.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:43.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:43.992 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:44.094 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:44.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:44.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[104516]: pgmap v847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 23634/56901 objects degraded (41.535%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[104516]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23634/56901 objects degraded (41.535%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:44.237 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:43] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.51.0" 2024-08-17T17:33:44.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[109246]: pgmap v847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 23634/56901 objects degraded (41.535%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:44.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:44.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:44.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[109246]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:33:44.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:44 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23634/56901 objects degraded (41.535%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:44.247 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:44.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:44.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:44.503 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:44.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:44.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:44 smithi175 ceph-mon[101434]: pgmap v847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 23634/56901 objects degraded (41.535%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:44 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:44 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:44 smithi175 ceph-mon[101434]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:33:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:44 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23634/56901 objects degraded (41.535%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:44.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:44.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:44.926 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:44.954 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:45.056 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:45.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:45.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:45.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:45.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:45.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:45.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:45.472 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:45.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:45.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.620 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:45.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:45.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:45.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:45.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:45.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:45.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:46.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:46.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:46.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:46.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:46.174 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:33:46.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:46.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:46.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[109246]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 23509/49917 objects degraded (47.096%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:46.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4195762990' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[104516]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 23509/49917 objects degraded (47.096%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4195762990' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:46.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:46.594 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:46 smithi175 ceph-mon[101434]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 23509/49917 objects degraded (47.096%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4195762990' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:33:46.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.723 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:46.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:46.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:46.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.875 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:46.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:46.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:47.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:47.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:47.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:47.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:33:47.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[104516]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:33:47.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:33:47.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[109246]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:33:47.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:47 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.602 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:47.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:33:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:33:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:47 smithi175 ceph-mon[101434]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:33:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:47 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:47.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:47.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:47.831 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:47.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:47.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:47.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:48.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:48.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:48.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:48.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:48.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:48.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:48.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[104516]: pgmap v851: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 23509/47034 objects degraded (49.983%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:48.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:48.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:48.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:48.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[109246]: pgmap v851: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 23509/47034 objects degraded (49.983%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:48.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:48.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:48.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:48.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:48 smithi175 ceph-mon[101434]: pgmap v851: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 23509/47034 objects degraded (49.983%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:48.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:48.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:48.752 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:33:48.753 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:48.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:33:48.790 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:48.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:48.827 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:33:48.827 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:48.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.846 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:48.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:48.863 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:33:48.863 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:48.877 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:48.877 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:48.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:33:48.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:48.929 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:33:48.929 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:48.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:48.961 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:33:48.961 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:48.971 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:48.996 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:33:48.996 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:49.073 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:49.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:49.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:49.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:49.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:49 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23509/47034 objects degraded (49.983%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3934793487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:49 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23509/47034 objects degraded (49.983%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3934793487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:49.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:49.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:49.604 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:49.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:49 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23509/47034 objects degraded (49.983%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3934793487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:49.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:49.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:50.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:50.028 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:50.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:50.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:50.154 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:50.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:50.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:33:50.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:50.327 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:50.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:50 smithi064 ceph-mon[109246]: pgmap v852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 23509/47034 objects degraded (49.983%); 1000 KiB/s, 15 objects/s recovering 2024-08-17T17:33:50.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:50 smithi064 ceph-mon[104516]: pgmap v852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 23509/47034 objects degraded (49.983%); 1000 KiB/s, 15 objects/s recovering 2024-08-17T17:33:50.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:50.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:50.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:33:50.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:50.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:50 smithi175 ceph-mon[101434]: pgmap v852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 983 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 23509/47034 objects degraded (49.983%); 1000 KiB/s, 15 objects/s recovering 2024-08-17T17:33:50.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:50.762 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:50.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:50.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:50.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:50.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:51.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:51.237 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:51.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:51.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:51.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:51.440 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:51.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:51.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:51.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:51.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:51.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:51.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:51.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:51.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:51.990 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.146 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:52.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:52.149 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:52.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.381 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:52 smithi064 ceph-mon[104516]: pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 809 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 23384/38706 objects degraded (60.414%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:52.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:52 smithi064 ceph-mon[109246]: pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 809 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 23384/38706 objects degraded (60.414%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:52.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:52.426 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:52.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:52.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:52 smithi175 ceph-mon[101434]: pgmap v853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 809 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 23384/38706 objects degraded (60.414%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:33:52.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:52.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:52.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:52.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:53.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:53.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:53.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:53.380 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.535 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.637 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:53.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:53.844 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:53.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:54.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:54.108 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:54.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:54.193 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:33:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:33:53] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.51.0" 2024-08-17T17:33:54.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:54.356 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:54.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:54.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:54.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:54.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:54 smithi064 ceph-mon[104516]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 785 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 23384/37563 objects degraded (62.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:54.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:54 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23384/37563 objects degraded (62.253%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:54 smithi064 ceph-mon[109246]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 785 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 23384/37563 objects degraded (62.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:54.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:54 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23384/37563 objects degraded (62.253%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:54.660 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:54 smithi175 ceph-mon[101434]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 785 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 23384/37563 objects degraded (62.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:33:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:54 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23384/37563 objects degraded (62.253%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:54.762 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:54.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:54.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:54.923 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:55.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:55.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:55.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:55.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:55.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:55.238 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:33:55.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:55.391 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:188401 7.e 7:71003fe9:::benchmark_data_smithi064_210574_object89592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:55.228352+0000", 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0051220340000000001, 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051528429999999998, 2024-08-17T17:33:55.392 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188401 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228352+0000", 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228352+0000", 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:33:55.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228354+0000", 2024-08-17T17:33:55.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.494e-06 2024-08-17T17:33:55.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228360+0000", 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7259999999999999e-06 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228376+0000", 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6082999999999998e-05 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:33:55.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228384+0000", 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8339999999999999e-06 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228442+0000", 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8851000000000003e-05 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228544+0000", 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101162 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228640+0000", 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6584999999999998e-05 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.401 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35217.0:188400 7.5 e267/228 7:af89feb0:::benchmark_data_smithi064_210574_object89591:head v 267'11781, mlcod=267'11779)", 2024-08-17T17:33:55.401 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:55.228685+0000", 2024-08-17T17:33:55.401 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0047898710000000002, 2024-08-17T17:33:55.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048892299999999996, 2024-08-17T17:33:55.401 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228685+0000", 2024-08-17T17:33:55.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228685+0000", 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228686+0000", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.034e-06 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228692+0000", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9149999999999999e-06 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228708+0000", 2024-08-17T17:33:55.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5031000000000001e-05 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228715+0000", 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7519999999999996e-06 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228876+0000", 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016114699999999999 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.228915+0000", 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8525000000000001e-05 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35217.0:188401 7.e e267/242)", 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:55.233195+0000", 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00027976699999999999, 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000424375, 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233195+0000", 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:33:55.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233195+0000", 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233197+0000", 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.889e-06 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233205+0000", 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1009999999999992e-06 2024-08-17T17:33:55.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233224+0000", 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9857e-05 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:55.233234+0000", 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7399999999999999e-06 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.409 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:33:55.410 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.410 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:33:55.410 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:33:55.410 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 3 2024-08-17T17:33:55.410 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:33:55.473 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:55.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:55.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:55.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:55.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:55.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:33:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:33:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:33:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:33:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:55.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:55.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:55.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:55.912 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:55.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:56.014 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:56.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:56.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:56.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:56.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:56.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:56.320 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:56.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[104516]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 659 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23288/31521 objects degraded (73.881%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:33:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[104516]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:33:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[109246]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 659 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23288/31521 objects degraded (73.881%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:33:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:56 smithi064 ceph-mon[109246]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:33:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:56.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:56.520 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:33:56.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:33:56.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:56 smithi175 ceph-mon[101434]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 659 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 23288/31521 objects degraded (73.881%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:56 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3282156070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:33:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:56 smithi175 ceph-mon[101434]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:33:56.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:56.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:56.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:56.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:56.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:56.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:56.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:57.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:57.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:57.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:57.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:57.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:57.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:33:57.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:57.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:57.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:57.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:57.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:57.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:57.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:57.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:57.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:57.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:58.047 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:58.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:58.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:58.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:58.253 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:33:58.258 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:58.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:58.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:58.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:58.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:58.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:58.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:58 smithi064 ceph-mon[104516]: pgmap v857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 598 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 23288/28572 objects degraded (81.506%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:58.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:58 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:58.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:58 smithi064 ceph-mon[109246]: pgmap v857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 598 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 23288/28572 objects degraded (81.506%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:58.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:58 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:58.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:58 smithi175 ceph-mon[101434]: pgmap v857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 598 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 23288/28572 objects degraded (81.506%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:33:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:58 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:58.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:58.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:58.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:58.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:59.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:59.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:59.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:59.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:59.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:59.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:59.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23288/28572 objects degraded (81.506%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[104516]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23288/28572 objects degraded (81.506%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:33:59 smithi064 ceph-mon[109246]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:33:59.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:59.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:59.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:33:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:59 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:59 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23288/28572 objects degraded (81.506%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1229196892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:33:59 smithi175 ceph-mon[101434]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:33:59.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:59.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:59.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:59.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:33:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:59.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.017 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:00.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.302 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:00.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:00.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:00 smithi175 ceph-mon[101434]: pgmap v859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 23155/26124 objects degraded (88.635%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:00.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:00 smithi064 ceph-mon[104516]: pgmap v859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 23155/26124 objects degraded (88.635%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:00.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:00 smithi064 ceph-mon[109246]: pgmap v859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 23155/26124 objects degraded (88.635%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:00.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:00.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.857 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.959 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:00.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:01.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:34:01.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:01.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:01.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:01.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:01.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:01.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:01.562 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:01.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:01.691 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:01.717 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:01.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:01.839 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:01.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:01.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:01.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:02.075 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:02.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:02.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.361 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:02.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:02 smithi175 ceph-mon[101434]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 421 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23155/20082 objects degraded (115.302%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:02 smithi064 ceph-mon[104516]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 421 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23155/20082 objects degraded (115.302%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:02 smithi064 ceph-mon[109246]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 421 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23155/20082 objects degraded (115.302%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:34:02.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:02.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:02.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:02.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:02.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:03.008 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:03.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:03.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:03.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:03.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:03.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 23155/20082 objects degraded (115.302%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 23155/20082 objects degraded (115.302%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 23155/20082 objects degraded (115.302%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:03.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:03.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:03.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:04.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:04.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:04.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:04.235 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.256 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:03] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-08-17T17:34:04.259 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.361 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:04.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:04.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:04.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:04.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:04 smithi175 ceph-mon[101434]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 397 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 23155/18921 objects degraded (122.377%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:34:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:04 smithi064 ceph-mon[104516]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 397 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 23155/18921 objects degraded (122.377%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:34:04.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:04 smithi064 ceph-mon[109246]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 397 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 23155/18921 objects degraded (122.377%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:34:04.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:04.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:04.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:04.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:05.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:05.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:05.178 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:05.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:05.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:05.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:05.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:05.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:05.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:05.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:05.828 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:05.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:05.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:05.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:06.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:06.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:34:06.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:06.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:06.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:06.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:06.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:06 smithi064 ceph-mon[104516]: pgmap v862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 262 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 22997/12465 objects degraded (184.493%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:06 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1868535831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:06 smithi064 ceph-mon[109246]: pgmap v862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 262 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 22997/12465 objects degraded (184.493%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:06.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:06 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1868535831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:06.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:06.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:06.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:06.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:06.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:06 smithi175 ceph-mon[101434]: pgmap v862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 262 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 22997/12465 objects degraded (184.493%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:06.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:06 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1868535831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:06.716 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:06.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:06.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:06.868 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:06.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:06.995 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:07.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:07.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:07.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:07.227 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:07.253 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:07.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:07.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:07.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:07.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:07.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:07.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:07.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:07.839 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:07.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:07.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:07.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:08.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:08.142 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:08.244 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:08.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:08.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:08.396 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:08.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:08.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:08.542 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:08.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:08.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:08.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:08 smithi175 ceph-mon[101434]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 203 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 22997/9627 objects degraded (238.880%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 22997/9627 objects degraded (238.880%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:08.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:08.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:08 smithi064 ceph-mon[104516]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 203 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 22997/9627 objects degraded (238.880%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 22997/9627 objects degraded (238.880%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:08 smithi064 ceph-mon[109246]: pgmap v863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 203 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 22997/9627 objects degraded (238.880%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:34:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 22997/9627 objects degraded (238.880%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:08.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.815 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:08.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:08.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:08.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:09.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:09.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:09.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:09.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:09.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:09.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:09.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:09.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:09.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:09.634 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:09.661 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 98806 objects 2024-08-17T17:34:09.662 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :147.856 2024-08-17T17:34:09.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:09.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:09.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:09.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:10.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:10.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:10.127 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:10.229 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:10.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:10.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:10.376 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:10.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:10.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:10.601 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:10.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:10 smithi175 ceph-mon[101434]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 152 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 22827/7164 objects degraded (318.635%); 2.0 MiB/s, 42 objects/s recovering 2024-08-17T17:34:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[104516]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 152 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 22827/7164 objects degraded (318.635%); 2.0 MiB/s, 42 objects/s recovering 2024-08-17T17:34:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[109246]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 152 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 22827/7164 objects degraded (318.635%); 2.0 MiB/s, 42 objects/s recovering 2024-08-17T17:34:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:10.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.746 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:10.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:10.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:10.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:10.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:11.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:11.115 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:11.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:11.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:11.257 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:11.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:11.363 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:11.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:11.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.612 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:11.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:11.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:11.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:11.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:11.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:12.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:12.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:12.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:12.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:12.228 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:12.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:12.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:12.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:12.535 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:12.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:12.637 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:12.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:12 smithi175 ceph-mon[101434]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 33 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 22827/1446 objects degraded (1578.631%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:12.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:12 smithi064 ceph-mon[104516]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 33 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 22827/1446 objects degraded (1578.631%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:12 smithi064 ceph-mon[109246]: pgmap v865: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 33 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 22827/1446 objects degraded (1578.631%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:12.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:12.781 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:12.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:12.906 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:13.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:13.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:13.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:13.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:13.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:34:13.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:13.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:13.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:13.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:13.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:13.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:13.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:13.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:13.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:13.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:13.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 22827/1446 objects degraded (1578.631%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:13.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 22827/1446 objects degraded (1578.631%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:13.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 22827/1446 objects degraded (1578.631%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:13.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:13.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:13.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.143 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:14.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:14.349 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:13] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:34:14.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:14.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:14.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:14 smithi175 ceph-mon[101434]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 22827/597 objects degraded (3823.618%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:14 smithi064 ceph-mon[104516]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 22827/597 objects degraded (3823.618%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:14.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:14 smithi064 ceph-mon[109246]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 22827/597 objects degraded (3823.618%); 1.1 MiB/s, 27 objects/s recovering 2024-08-17T17:34:14.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:14.755 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-08-17T17:34:14.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-08-17T17:34:14.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:14.865 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.979 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:34:14.980 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:34:14.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:15.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:15.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.137 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:15.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:15.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:15.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:15.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:15.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:15.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:15.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:15.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:15 smithi175 ceph-mon[101434]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:15 smithi064 ceph-mon[104516]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:15 smithi064 ceph-mon[109246]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:15.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:15.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:15.884 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:15.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:15.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:16.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:16.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:16.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:16.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:16.254 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:16.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:16.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:16.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:16.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:16.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:16.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:16.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:16 smithi175 ceph-mon[101434]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 22277/84 objects degraded (26520.238%); 1.1 MiB/s, 73 objects/s recovering 2024-08-17T17:34:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:16 smithi064 ceph-mon[104516]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 22277/84 objects degraded (26520.238%); 1.1 MiB/s, 73 objects/s recovering 2024-08-17T17:34:16.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:16 smithi064 ceph-mon[109246]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 22277/84 objects degraded (26520.238%); 1.1 MiB/s, 73 objects/s recovering 2024-08-17T17:34:16.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:16.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:16.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:16.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:16.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:16.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:17.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:17.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:17.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:17.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:17.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:17.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:17.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:17.617 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:17.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:17.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:17.863 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:17.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:17.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:18.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:18.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:18.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:18.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:18.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:18.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.359 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:18.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:18.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:18.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:18.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:18.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:18.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:18 smithi175 ceph-mon[101434]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 22277/84 objects degraded (26520.238%); 288 KiB/s, 59 objects/s recovering 2024-08-17T17:34:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 22277/84 objects degraded (26520.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:18 smithi064 ceph-mon[104516]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 22277/84 objects degraded (26520.238%); 288 KiB/s, 59 objects/s recovering 2024-08-17T17:34:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 22277/84 objects degraded (26520.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:18 smithi064 ceph-mon[109246]: pgmap v868: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 22277/84 objects degraded (26520.238%); 288 KiB/s, 59 objects/s recovering 2024-08-17T17:34:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 22277/84 objects degraded (26520.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:18.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:18.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:18.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:18.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:18.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:18.979 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:19.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:19.105 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:19.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:19.229 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":866,"stamp":"2024-08-17T17:34:13.103371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":15996901,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":198566,"num_write_kb":6338626,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7379,"num_bytes_recovered":466689859,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138425,"ondisk_log_size":138425,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3275280,"kb_used_data":103248,"kb_used_omap":46,"kb_used_meta":3171857,"kb_avail":746685936,"statfs":{"total":767960285184,"available":764606398464,"internally_reserved":0,"allocated":105725952,"data_stored":94152042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":3247981811},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-425590808,"num_objects":-6495,"num_object_clones":0,"num_object_copies":-19485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-328,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6495,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":13893632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003559"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":395,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.286018+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:33:58.286018+0000","last_peered":"2024-08-17T17:33:58.286018+0000","last_clean":"2024-08-17T17:33:58.286018+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:33:58.286018+0000","last_undegraded":"2024-08-17T17:33:58.286018+0000","last_fullsized":"2024-08-17T17:33:58.286018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":266,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238420+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:33:55.238420+0000","last_peered":"2024-08-17T17:33:55.238420+0000","last_clean":"2024-08-17T17:33:55.238420+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:33:55.238420+0000","last_undegraded":"2024-08-17T17:33:55.238420+0000","last_fullsized":"2024-08-17T17:33:55.238420+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252491+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:33:55.252491+0000","last_peered":"2024-08-17T17:33:55.252491+0000","last_clean":"2024-08-17T17:33:55.252491+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:33:55.252491+0000","last_undegraded":"2024-08-17T17:33:55.252491+0000","last_fullsized":"2024-08-17T17:33:55.252491+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":387,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.298135+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:33:58.298135+0000","last_peered":"2024-08-17T17:33:58.298135+0000","last_clean":"2024-08-17T17:33:58.298135+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:33:58.298135+0000","last_undegraded":"2024-08-17T17:33:58.298135+0000","last_fullsized":"2024-08-17T17:33:58.298135+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":386,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.304459+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:33:58.304459+0000","last_peered":"2024-08-17T17:33:58.304459+0000","last_clean":"2024-08-17T17:33:58.304459+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:33:58.304459+0000","last_undegraded":"2024-08-17T17:33:58.304459+0000","last_fullsized":"2024-08-17T17:33:58.304459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":252,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238263+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:33:55.238263+0000","last_peered":"2024-08-17T17:33:55.238263+0000","last_clean":"2024-08-17T17:33:55.238263+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:33:55.238263+0000","last_undegraded":"2024-08-17T17:33:55.238263+0000","last_fullsized":"2024-08-17T17:33:55.238263+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":395,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.294663+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:33:58.294663+0000","last_peered":"2024-08-17T17:33:58.294663+0000","last_clean":"2024-08-17T17:33:58.294663+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:33:58.294663+0000","last_undegraded":"2024-08-17T17:33:58.294663+0000","last_fullsized":"2024-08-17T17:33:58.294663+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":200,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.239132+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:33:55.239132+0000","last_peered":"2024-08-17T17:33:55.239132+0000","last_clean":"2024-08-17T17:33:55.239132+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:33:55.239132+0000","last_undegraded":"2024-08-17T17:33:55.239132+0000","last_fullsized":"2024-08-17T17:33:55.239132+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"269'12450","reported_seq":12551,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:09.642308+0000","last_change":"2024-08-17T17:31:37.408577+0000","last_active":"2024-08-17T17:34:09.642308+0000","last_peered":"2024-08-17T17:34:09.642308+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.396040+0000","last_became_peered":"2024-08-17T17:31:37.396040+0000","last_unstale":"2024-08-17T17:34:09.642308+0000","last_undegraded":"2024-08-17T17:31:37.194642+0000","last_fullsized":"2024-08-17T17:31:37.151313+0000","mapping_epoch":248,"log_start":"236'4500","ondisk_log_start":"236'4500","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7950,"log_dups_size":0,"ondisk_log_size":7950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12450,"num_write_kb":398400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-6063}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":261,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238713+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:33:55.238713+0000","last_peered":"2024-08-17T17:33:55.238713+0000","last_clean":"2024-08-17T17:33:55.238713+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:33:55.238713+0000","last_undegraded":"2024-08-17T17:33:55.238713+0000","last_fullsized":"2024-08-17T17:33:55.238713+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252104+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:33:55.252104+0000","last_peered":"2024-08-17T17:33:55.252104+0000","last_clean":"2024-08-17T17:33:55.252104+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:33:55.252104+0000","last_undegraded":"2024-08-17T17:33:55.252104+0000","last_fullsized":"2024-08-17T17:33:55.252104+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"269'12235","reported_seq":12375,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:08.534913+0000","last_change":"2024-08-17T17:31:17.798927+0000","last_active":"2024-08-17T17:34:08.534913+0000","last_peered":"2024-08-17T17:34:08.534913+0000","last_clean":"2024-08-17T17:34:08.534913+0000","last_became_active":"2024-08-17T17:31:17.603128+0000","last_became_peered":"2024-08-17T17:31:17.603128+0000","last_unstale":"2024-08-17T17:34:08.534913+0000","last_undegraded":"2024-08-17T17:34:08.534913+0000","last_fullsized":"2024-08-17T17:34:08.534913+0000","mapping_epoch":242,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:37.753356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":262,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238596+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:33:55.238596+0000","last_peered":"2024-08-17T17:33:55.238596+0000","last_clean":"2024-08-17T17:33:55.238596+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:33:55.238596+0000","last_undegraded":"2024-08-17T17:33:55.238596+0000","last_fullsized":"2024-08-17T17:33:55.238596+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259509+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:33:55.259509+0000","last_peered":"2024-08-17T17:33:55.259509+0000","last_clean":"2024-08-17T17:33:55.259509+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:33:55.259509+0000","last_undegraded":"2024-08-17T17:33:55.259509+0000","last_fullsized":"2024-08-17T17:33:55.259509+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"269'12526","reported_seq":12620,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.525618+0000","last_change":"2024-08-17T17:29:57.202283+0000","last_active":"2024-08-17T17:34:09.525618+0000","last_peered":"2024-08-17T17:34:09.525618+0000","last_clean":"2024-08-17T17:34:09.525618+0000","last_became_active":"2024-08-17T17:29:57.201797+0000","last_became_peered":"2024-08-17T17:29:57.201797+0000","last_unstale":"2024-08-17T17:34:09.525618+0000","last_undegraded":"2024-08-17T17:34:09.525618+0000","last_fullsized":"2024-08-17T17:34:09.525618+0000","mapping_epoch":228,"log_start":"235'3700","ondisk_log_start":"235'3700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8826,"log_dups_size":0,"ondisk_log_size":8826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:31.699096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12526,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":279,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238559+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:33:55.238559+0000","last_peered":"2024-08-17T17:33:55.238559+0000","last_clean":"2024-08-17T17:33:55.238559+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:33:55.238559+0000","last_undegraded":"2024-08-17T17:33:55.238559+0000","last_fullsized":"2024-08-17T17:33:55.238559+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":180,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.239605+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:33:55.239605+0000","last_peered":"2024-08-17T17:33:55.239605+0000","last_clean":"2024-08-17T17:33:55.239605+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:33:55.239605+0000","last_undegraded":"2024-08-17T17:33:55.239605+0000","last_fullsized":"2024-08-17T17:33:55.239605+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"269'12310","reported_seq":12405,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.606812+0000","last_change":"2024-08-17T17:29:57.200346+0000","last_active":"2024-08-17T17:34:09.606812+0000","last_peered":"2024-08-17T17:34:09.606812+0000","last_clean":"2024-08-17T17:34:09.606812+0000","last_became_active":"2024-08-17T17:29:57.200066+0000","last_became_peered":"2024-08-17T17:29:57.200066+0000","last_unstale":"2024-08-17T17:34:09.606812+0000","last_undegraded":"2024-08-17T17:34:09.606812+0000","last_fullsized":"2024-08-17T17:34:09.606812+0000","mapping_epoch":228,"log_start":"234'2300","ondisk_log_start":"234'2300","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:31.030646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12310,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252286+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:33:55.252286+0000","last_peered":"2024-08-17T17:33:55.252286+0000","last_clean":"2024-08-17T17:33:55.252286+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:33:55.252286+0000","last_undegraded":"2024-08-17T17:33:55.252286+0000","last_fullsized":"2024-08-17T17:33:55.252286+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":401,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.298628+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:33:58.298628+0000","last_peered":"2024-08-17T17:33:58.298628+0000","last_clean":"2024-08-17T17:33:58.298628+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:33:58.298628+0000","last_undegraded":"2024-08-17T17:33:58.298628+0000","last_fullsized":"2024-08-17T17:33:58.298628+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"269'12371","reported_seq":12465,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.439755+0000","last_change":"2024-08-17T17:29:57.204892+0000","last_active":"2024-08-17T17:34:09.439755+0000","last_peered":"2024-08-17T17:34:09.439755+0000","last_clean":"2024-08-17T17:34:09.439755+0000","last_became_active":"2024-08-17T17:29:57.204591+0000","last_became_peered":"2024-08-17T17:29:57.204591+0000","last_unstale":"2024-08-17T17:34:09.439755+0000","last_undegraded":"2024-08-17T17:34:09.439755+0000","last_fullsized":"2024-08-17T17:34:09.439755+0000","mapping_epoch":228,"log_start":"234'3500","ondisk_log_start":"234'3500","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8871,"log_dups_size":0,"ondisk_log_size":8871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:01:58.087057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12371,"num_write_kb":396096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":416,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.291552+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:33:58.291552+0000","last_peered":"2024-08-17T17:33:58.291552+0000","last_clean":"2024-08-17T17:33:58.291552+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:33:58.291552+0000","last_undegraded":"2024-08-17T17:33:58.291552+0000","last_fullsized":"2024-08-17T17:33:58.291552+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.262661+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:33:55.262661+0000","last_peered":"2024-08-17T17:33:55.262661+0000","last_clean":"2024-08-17T17:33:55.262661+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:33:55.262661+0000","last_undegraded":"2024-08-17T17:33:55.262661+0000","last_fullsized":"2024-08-17T17:33:55.262661+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"269'12506","reported_seq":12643,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.601460+0000","last_change":"2024-08-17T17:31:17.661359+0000","last_active":"2024-08-17T17:34:09.601460+0000","last_peered":"2024-08-17T17:34:09.601460+0000","last_clean":"2024-08-17T17:34:09.601460+0000","last_became_active":"2024-08-17T17:31:17.599056+0000","last_became_peered":"2024-08-17T17:31:17.599056+0000","last_unstale":"2024-08-17T17:34:09.601460+0000","last_undegraded":"2024-08-17T17:34:09.601460+0000","last_fullsized":"2024-08-17T17:34:09.601460+0000","mapping_epoch":242,"log_start":"234'2500","ondisk_log_start":"234'2500","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:12.648106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12506,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284376+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:33:58.284376+0000","last_peered":"2024-08-17T17:33:58.284376+0000","last_clean":"2024-08-17T17:33:58.284376+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:33:58.284376+0000","last_undegraded":"2024-08-17T17:33:58.284376+0000","last_fullsized":"2024-08-17T17:33:58.284376+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":157,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259261+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:33:55.259261+0000","last_peered":"2024-08-17T17:33:55.259261+0000","last_clean":"2024-08-17T17:33:55.259261+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:33:55.259261+0000","last_undegraded":"2024-08-17T17:33:55.259261+0000","last_fullsized":"2024-08-17T17:33:55.259261+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"269'12360","reported_seq":12455,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:09.625923+0000","last_change":"2024-08-17T17:31:37.408160+0000","last_active":"2024-08-17T17:34:09.625923+0000","last_peered":"2024-08-17T17:34:09.625923+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.395243+0000","last_became_peered":"2024-08-17T17:31:37.395243+0000","last_unstale":"2024-08-17T17:34:09.625923+0000","last_undegraded":"2024-08-17T17:31:37.192943+0000","last_fullsized":"2024-08-17T17:31:37.149514+0000","mapping_epoch":248,"log_start":"235'4400","ondisk_log_start":"235'4400","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12360,"num_write_kb":395457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":-6013}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259433+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:33:55.259433+0000","last_peered":"2024-08-17T17:33:55.259433+0000","last_clean":"2024-08-17T17:33:55.259433+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:33:55.259433+0000","last_undegraded":"2024-08-17T17:33:55.259433+0000","last_fullsized":"2024-08-17T17:33:55.259433+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259189+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:33:55.259189+0000","last_peered":"2024-08-17T17:33:55.259189+0000","last_clean":"2024-08-17T17:33:55.259189+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:33:55.259189+0000","last_undegraded":"2024-08-17T17:33:55.259189+0000","last_fullsized":"2024-08-17T17:33:55.259189+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"269'12272","reported_seq":14953,"reported_epoch":269,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:08.595846+0000","last_change":"2024-08-17T17:33:21.287503+0000","last_active":"2024-08-17T17:34:08.595846+0000","last_peered":"2024-08-17T17:34:08.595846+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.358516+0000","last_became_peered":"2024-08-17T17:31:37.358516+0000","last_unstale":"2024-08-17T17:34:08.595846+0000","last_undegraded":"2024-08-17T17:31:37.192795+0000","last_fullsized":"2024-08-17T17:31:37.150416+0000","mapping_epoch":248,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1703936,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12272,"num_write_kb":393536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1294,"num_bytes_recovered":77201408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-4686}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":397,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.262621+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:33:58.262621+0000","last_peered":"2024-08-17T17:33:58.262621+0000","last_clean":"2024-08-17T17:33:58.262621+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:33:58.262621+0000","last_undegraded":"2024-08-17T17:33:58.262621+0000","last_fullsized":"2024-08-17T17:33:58.262621+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":385,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.272289+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:33:58.272289+0000","last_peered":"2024-08-17T17:33:58.272289+0000","last_clean":"2024-08-17T17:33:58.272289+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:33:58.272289+0000","last_undegraded":"2024-08-17T17:33:58.272289+0000","last_fullsized":"2024-08-17T17:33:58.272289+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"269'12235","reported_seq":12328,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.550927+0000","last_change":"2024-08-17T17:29:57.204368+0000","last_active":"2024-08-17T17:34:09.550927+0000","last_peered":"2024-08-17T17:34:09.550927+0000","last_clean":"2024-08-17T17:34:09.550927+0000","last_became_active":"2024-08-17T17:29:57.204070+0000","last_became_peered":"2024-08-17T17:29:57.204070+0000","last_unstale":"2024-08-17T17:34:09.550927+0000","last_undegraded":"2024-08-17T17:34:09.550927+0000","last_fullsized":"2024-08-17T17:34:09.550927+0000","mapping_epoch":228,"log_start":"234'3400","ondisk_log_start":"234'3400","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8835,"log_dups_size":0,"ondisk_log_size":8835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.001063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":582,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293670+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:33:58.293670+0000","last_peered":"2024-08-17T17:33:58.293670+0000","last_clean":"2024-08-17T17:33:58.293670+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:33:58.293670+0000","last_undegraded":"2024-08-17T17:33:58.293670+0000","last_fullsized":"2024-08-17T17:33:58.293670+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":858,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252365+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:33:55.252365+0000","last_peered":"2024-08-17T17:33:55.252365+0000","last_clean":"2024-08-17T17:33:55.252365+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:33:55.252365+0000","last_undegraded":"2024-08-17T17:33:55.252365+0000","last_fullsized":"2024-08-17T17:33:55.252365+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":380,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.291995+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:33:58.291995+0000","last_peered":"2024-08-17T17:33:58.291995+0000","last_clean":"2024-08-17T17:33:58.291995+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:33:58.291995+0000","last_undegraded":"2024-08-17T17:33:58.291995+0000","last_fullsized":"2024-08-17T17:33:58.291995+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":171,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259140+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:33:55.259140+0000","last_peered":"2024-08-17T17:33:55.259140+0000","last_clean":"2024-08-17T17:33:55.259140+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:33:55.259140+0000","last_undegraded":"2024-08-17T17:33:55.259140+0000","last_fullsized":"2024-08-17T17:33:55.259140+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"269'12350","reported_seq":12401,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.612780+0000","last_change":"2024-08-17T17:29:57.203033+0000","last_active":"2024-08-17T17:34:09.612780+0000","last_peered":"2024-08-17T17:34:09.612780+0000","last_clean":"2024-08-17T17:34:09.612780+0000","last_became_active":"2024-08-17T17:29:57.202766+0000","last_became_peered":"2024-08-17T17:29:57.202766+0000","last_unstale":"2024-08-17T17:34:09.612780+0000","last_undegraded":"2024-08-17T17:34:09.612780+0000","last_fullsized":"2024-08-17T17:34:09.612780+0000","mapping_epoch":228,"log_start":"235'3700","ondisk_log_start":"235'3700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8650,"log_dups_size":0,"ondisk_log_size":8650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:06.312698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12350,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":644,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238276+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:33:55.238276+0000","last_peered":"2024-08-17T17:33:55.238276+0000","last_clean":"2024-08-17T17:33:55.238276+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:33:55.238276+0000","last_undegraded":"2024-08-17T17:33:55.238276+0000","last_fullsized":"2024-08-17T17:33:55.238276+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":402,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293731+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:33:58.293731+0000","last_peered":"2024-08-17T17:33:58.293731+0000","last_clean":"2024-08-17T17:33:58.293731+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:33:58.293731+0000","last_undegraded":"2024-08-17T17:33:58.293731+0000","last_fullsized":"2024-08-17T17:33:58.293731+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":398,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.304494+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:33:58.304494+0000","last_peered":"2024-08-17T17:33:58.304494+0000","last_clean":"2024-08-17T17:33:58.304494+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:33:58.304494+0000","last_undegraded":"2024-08-17T17:33:58.304494+0000","last_fullsized":"2024-08-17T17:33:58.304494+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"269'12266","reported_seq":12361,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.556674+0000","last_change":"2024-08-17T17:29:57.202172+0000","last_active":"2024-08-17T17:34:09.556674+0000","last_peered":"2024-08-17T17:34:09.556674+0000","last_clean":"2024-08-17T17:34:09.556674+0000","last_became_active":"2024-08-17T17:29:57.201785+0000","last_became_peered":"2024-08-17T17:29:57.201785+0000","last_unstale":"2024-08-17T17:34:09.556674+0000","last_undegraded":"2024-08-17T17:34:09.556674+0000","last_fullsized":"2024-08-17T17:34:09.556674+0000","mapping_epoch":228,"log_start":"234'3400","ondisk_log_start":"234'3400","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:49.084764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12266,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":799,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.271847+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:33:58.271847+0000","last_peered":"2024-08-17T17:33:58.271847+0000","last_clean":"2024-08-17T17:33:58.271847+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:33:58.271847+0000","last_undegraded":"2024-08-17T17:33:58.271847+0000","last_fullsized":"2024-08-17T17:33:58.271847+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":391,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.288830+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:33:58.288830+0000","last_peered":"2024-08-17T17:33:58.288830+0000","last_clean":"2024-08-17T17:33:58.288830+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:33:58.288830+0000","last_undegraded":"2024-08-17T17:33:58.288830+0000","last_fullsized":"2024-08-17T17:33:58.288830+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":438,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.288645+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:33:58.288645+0000","last_peered":"2024-08-17T17:33:58.288645+0000","last_clean":"2024-08-17T17:33:58.288645+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:33:58.288645+0000","last_undegraded":"2024-08-17T17:33:58.288645+0000","last_fullsized":"2024-08-17T17:33:58.288645+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"269'12376","reported_seq":12468,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:09.521058+0000","last_change":"2024-08-17T17:31:37.399805+0000","last_active":"2024-08-17T17:34:09.521058+0000","last_peered":"2024-08-17T17:34:09.521058+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.386849+0000","last_became_peered":"2024-08-17T17:31:37.386849+0000","last_unstale":"2024-08-17T17:34:09.521058+0000","last_undegraded":"2024-08-17T17:31:37.193155+0000","last_fullsized":"2024-08-17T17:31:37.153842+0000","mapping_epoch":248,"log_start":"234'3500","ondisk_log_start":"234'3500","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6039,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12376,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-6033}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":981,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252296+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:33:55.252296+0000","last_peered":"2024-08-17T17:33:55.252296+0000","last_clean":"2024-08-17T17:33:55.252296+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:33:55.252296+0000","last_undegraded":"2024-08-17T17:33:55.252296+0000","last_fullsized":"2024-08-17T17:33:55.252296+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":216,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.272984+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:33:58.272984+0000","last_peered":"2024-08-17T17:33:58.272984+0000","last_clean":"2024-08-17T17:33:58.272984+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:33:58.272984+0000","last_undegraded":"2024-08-17T17:33:58.272984+0000","last_fullsized":"2024-08-17T17:33:58.272984+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":285,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.281294+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:33:58.281294+0000","last_peered":"2024-08-17T17:33:58.281294+0000","last_clean":"2024-08-17T17:33:58.281294+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:33:58.281294+0000","last_undegraded":"2024-08-17T17:33:58.281294+0000","last_fullsized":"2024-08-17T17:33:58.281294+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"269'12206","reported_seq":24138,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.603161+0000","last_change":"2024-08-17T17:33:22.708397+0000","last_active":"2024-08-17T17:34:09.603161+0000","last_peered":"2024-08-17T17:34:09.603161+0000","last_clean":"2024-08-17T17:34:09.603161+0000","last_became_active":"2024-08-17T17:33:22.677716+0000","last_became_peered":"2024-08-17T17:33:22.677716+0000","last_unstale":"2024-08-17T17:34:09.603161+0000","last_undegraded":"2024-08-17T17:34:09.603161+0000","last_fullsized":"2024-08-17T17:34:09.603161+0000","mapping_epoch":263,"log_start":"235'4000","ondisk_log_start":"235'4000","created":228,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:23.405360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5932,"num_bytes_recovered":376832000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":50,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252320+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:33:55.252320+0000","last_peered":"2024-08-17T17:33:55.252320+0000","last_clean":"2024-08-17T17:33:55.252320+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:33:55.252320+0000","last_undegraded":"2024-08-17T17:33:55.252320+0000","last_fullsized":"2024-08-17T17:33:55.252320+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":683,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.281507+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:33:58.281507+0000","last_peered":"2024-08-17T17:33:58.281507+0000","last_clean":"2024-08-17T17:33:58.281507+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:33:58.281507+0000","last_undegraded":"2024-08-17T17:33:58.281507+0000","last_fullsized":"2024-08-17T17:33:58.281507+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":407,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.280577+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:33:58.280577+0000","last_peered":"2024-08-17T17:33:58.280577+0000","last_clean":"2024-08-17T17:33:58.280577+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:33:58.280577+0000","last_undegraded":"2024-08-17T17:33:58.280577+0000","last_fullsized":"2024-08-17T17:33:58.280577+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"269'12426","reported_seq":12491,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.619815+0000","last_change":"2024-08-17T17:31:17.646602+0000","last_active":"2024-08-17T17:34:09.619815+0000","last_peered":"2024-08-17T17:34:09.619815+0000","last_clean":"2024-08-17T17:34:09.619815+0000","last_became_active":"2024-08-17T17:31:17.591931+0000","last_became_peered":"2024-08-17T17:31:17.591931+0000","last_unstale":"2024-08-17T17:34:09.619815+0000","last_undegraded":"2024-08-17T17:34:09.619815+0000","last_fullsized":"2024-08-17T17:34:09.619815+0000","mapping_epoch":242,"log_start":"234'2400","ondisk_log_start":"234'2400","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:12:29.536081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12426,"num_write_kb":397632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":404,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.281398+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:33:58.281398+0000","last_peered":"2024-08-17T17:33:58.281398+0000","last_clean":"2024-08-17T17:33:58.281398+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:33:58.281398+0000","last_undegraded":"2024-08-17T17:33:58.281398+0000","last_fullsized":"2024-08-17T17:33:58.281398+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":834,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238589+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:33:55.238589+0000","last_peered":"2024-08-17T17:33:55.238589+0000","last_clean":"2024-08-17T17:33:55.238589+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:33:55.238589+0000","last_undegraded":"2024-08-17T17:33:55.238589+0000","last_fullsized":"2024-08-17T17:33:55.238589+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":402,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.298581+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:33:58.298581+0000","last_peered":"2024-08-17T17:33:58.298581+0000","last_clean":"2024-08-17T17:33:58.298581+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:33:58.298581+0000","last_undegraded":"2024-08-17T17:33:58.298581+0000","last_fullsized":"2024-08-17T17:33:58.298581+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"269'12238","reported_seq":12330,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:08.594151+0000","last_change":"2024-08-17T17:29:57.198724+0000","last_active":"2024-08-17T17:34:08.594151+0000","last_peered":"2024-08-17T17:34:08.594151+0000","last_clean":"2024-08-17T17:34:08.594151+0000","last_became_active":"2024-08-17T17:29:57.198432+0000","last_became_peered":"2024-08-17T17:29:57.198432+0000","last_unstale":"2024-08-17T17:34:08.594151+0000","last_undegraded":"2024-08-17T17:34:08.594151+0000","last_fullsized":"2024-08-17T17:34:08.594151+0000","mapping_epoch":228,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:23:54.902580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12238,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":395,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.294902+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:33:58.294902+0000","last_peered":"2024-08-17T17:33:58.294902+0000","last_clean":"2024-08-17T17:33:58.294902+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:33:58.294902+0000","last_undegraded":"2024-08-17T17:33:58.294902+0000","last_fullsized":"2024-08-17T17:33:58.294902+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":914,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.288471+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:33:58.288471+0000","last_peered":"2024-08-17T17:33:58.288471+0000","last_clean":"2024-08-17T17:33:58.288471+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:33:58.288471+0000","last_undegraded":"2024-08-17T17:33:58.288471+0000","last_fullsized":"2024-08-17T17:33:58.288471+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":392,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.258565+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:33:58.258565+0000","last_peered":"2024-08-17T17:33:58.258565+0000","last_clean":"2024-08-17T17:33:58.258565+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:33:58.258565+0000","last_undegraded":"2024-08-17T17:33:58.258565+0000","last_fullsized":"2024-08-17T17:33:58.258565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"269'12316","reported_seq":12411,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:08.574279+0000","last_change":"2024-08-17T17:29:57.199611+0000","last_active":"2024-08-17T17:34:08.574279+0000","last_peered":"2024-08-17T17:34:08.574279+0000","last_clean":"2024-08-17T17:34:08.574279+0000","last_became_active":"2024-08-17T17:29:57.199174+0000","last_became_peered":"2024-08-17T17:29:57.199174+0000","last_unstale":"2024-08-17T17:34:08.574279+0000","last_undegraded":"2024-08-17T17:34:08.574279+0000","last_fullsized":"2024-08-17T17:34:08.574279+0000","mapping_epoch":228,"log_start":"236'4800","ondisk_log_start":"236'4800","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7516,"log_dups_size":0,"ondisk_log_size":7516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:35.587097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12316,"num_write_kb":395456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":413,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.304297+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:33:58.304297+0000","last_peered":"2024-08-17T17:33:58.304297+0000","last_clean":"2024-08-17T17:33:58.304297+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:33:58.304297+0000","last_undegraded":"2024-08-17T17:33:58.304297+0000","last_fullsized":"2024-08-17T17:33:58.304297+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":390,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.294036+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:33:58.294036+0000","last_peered":"2024-08-17T17:33:58.294036+0000","last_clean":"2024-08-17T17:33:58.294036+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:33:58.294036+0000","last_undegraded":"2024-08-17T17:33:58.294036+0000","last_fullsized":"2024-08-17T17:33:58.294036+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259551+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:33:55.259551+0000","last_peered":"2024-08-17T17:33:55.259551+0000","last_clean":"2024-08-17T17:33:55.259551+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:33:55.259551+0000","last_undegraded":"2024-08-17T17:33:55.259551+0000","last_fullsized":"2024-08-17T17:33:55.259551+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.285085+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:33:58.285085+0000","last_peered":"2024-08-17T17:33:58.285085+0000","last_clean":"2024-08-17T17:33:58.285085+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:33:58.285085+0000","last_undegraded":"2024-08-17T17:33:58.285085+0000","last_fullsized":"2024-08-17T17:33:58.285085+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":182,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284570+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:33:58.284570+0000","last_peered":"2024-08-17T17:33:58.284570+0000","last_clean":"2024-08-17T17:33:58.284570+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:33:58.284570+0000","last_undegraded":"2024-08-17T17:33:58.284570+0000","last_fullsized":"2024-08-17T17:33:58.284570+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":390,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.280463+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:33:58.280463+0000","last_peered":"2024-08-17T17:33:58.280463+0000","last_clean":"2024-08-17T17:33:58.280463+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:33:58.280463+0000","last_undegraded":"2024-08-17T17:33:58.280463+0000","last_fullsized":"2024-08-17T17:33:58.280463+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":266,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.238513+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:33:55.238513+0000","last_peered":"2024-08-17T17:33:55.238513+0000","last_clean":"2024-08-17T17:33:55.238513+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:33:55.238513+0000","last_undegraded":"2024-08-17T17:33:55.238513+0000","last_fullsized":"2024-08-17T17:33:55.238513+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252448+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:33:55.252448+0000","last_peered":"2024-08-17T17:33:55.252448+0000","last_clean":"2024-08-17T17:33:55.252448+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:33:55.252448+0000","last_undegraded":"2024-08-17T17:33:55.252448+0000","last_fullsized":"2024-08-17T17:33:55.252448+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":357,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.285568+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:33:58.285568+0000","last_peered":"2024-08-17T17:33:58.285568+0000","last_clean":"2024-08-17T17:33:58.285568+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:33:58.285568+0000","last_undegraded":"2024-08-17T17:33:58.285568+0000","last_fullsized":"2024-08-17T17:33:58.285568+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":415,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.297940+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:33:58.297940+0000","last_peered":"2024-08-17T17:33:58.297940+0000","last_clean":"2024-08-17T17:33:58.297940+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:33:58.297940+0000","last_undegraded":"2024-08-17T17:33:58.297940+0000","last_fullsized":"2024-08-17T17:33:58.297940+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":406,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.258063+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:33:58.258063+0000","last_peered":"2024-08-17T17:33:58.258063+0000","last_clean":"2024-08-17T17:33:58.258063+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:33:58.258063+0000","last_undegraded":"2024-08-17T17:33:58.258063+0000","last_fullsized":"2024-08-17T17:33:58.258063+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252429+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:33:55.252429+0000","last_peered":"2024-08-17T17:33:55.252429+0000","last_clean":"2024-08-17T17:33:55.252429+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:33:55.252429+0000","last_undegraded":"2024-08-17T17:33:55.252429+0000","last_fullsized":"2024-08-17T17:33:55.252429+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":411,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.258729+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:33:58.258729+0000","last_peered":"2024-08-17T17:33:58.258729+0000","last_clean":"2024-08-17T17:33:58.258729+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:33:58.258729+0000","last_undegraded":"2024-08-17T17:33:58.258729+0000","last_fullsized":"2024-08-17T17:33:58.258729+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":388,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.304410+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:33:58.304410+0000","last_peered":"2024-08-17T17:33:58.304410+0000","last_clean":"2024-08-17T17:33:58.304410+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:33:58.304410+0000","last_undegraded":"2024-08-17T17:33:58.304410+0000","last_fullsized":"2024-08-17T17:33:58.304410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":392,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.287676+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:33:58.287676+0000","last_peered":"2024-08-17T17:33:58.287676+0000","last_clean":"2024-08-17T17:33:58.287676+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:33:58.287676+0000","last_undegraded":"2024-08-17T17:33:58.287676+0000","last_fullsized":"2024-08-17T17:33:58.287676+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":408,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.285576+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:33:58.285576+0000","last_peered":"2024-08-17T17:33:58.285576+0000","last_clean":"2024-08-17T17:33:58.285576+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:33:58.285576+0000","last_undegraded":"2024-08-17T17:33:58.285576+0000","last_fullsized":"2024-08-17T17:33:58.285576+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":156,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.259051+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:33:55.259051+0000","last_peered":"2024-08-17T17:33:55.259051+0000","last_clean":"2024-08-17T17:33:55.259051+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:33:55.259051+0000","last_undegraded":"2024-08-17T17:33:55.259051+0000","last_fullsized":"2024-08-17T17:33:55.259051+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":417,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.272052+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:33:58.272052+0000","last_peered":"2024-08-17T17:33:58.272052+0000","last_clean":"2024-08-17T17:33:58.272052+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:33:58.272052+0000","last_undegraded":"2024-08-17T17:33:58.272052+0000","last_fullsized":"2024-08-17T17:33:58.272052+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":393,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.298437+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:33:58.298437+0000","last_peered":"2024-08-17T17:33:58.298437+0000","last_clean":"2024-08-17T17:33:58.298437+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:33:58.298437+0000","last_undegraded":"2024-08-17T17:33:58.298437+0000","last_fullsized":"2024-08-17T17:33:58.298437+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":156,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.261655+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:33:55.261655+0000","last_peered":"2024-08-17T17:33:55.261655+0000","last_clean":"2024-08-17T17:33:55.261655+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:33:55.261655+0000","last_undegraded":"2024-08-17T17:33:55.261655+0000","last_fullsized":"2024-08-17T17:33:55.261655+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":54,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252213+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:33:55.252213+0000","last_peered":"2024-08-17T17:33:55.252213+0000","last_clean":"2024-08-17T17:33:55.252213+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:33:55.252213+0000","last_undegraded":"2024-08-17T17:33:55.252213+0000","last_fullsized":"2024-08-17T17:33:55.252213+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":156,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.258979+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:33:55.258979+0000","last_peered":"2024-08-17T17:33:55.258979+0000","last_clean":"2024-08-17T17:33:55.258979+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:33:55.258979+0000","last_undegraded":"2024-08-17T17:33:55.258979+0000","last_fullsized":"2024-08-17T17:33:55.258979+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":64,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.252402+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:33:55.252402+0000","last_peered":"2024-08-17T17:33:55.252402+0000","last_clean":"2024-08-17T17:33:55.252402+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:33:55.252402+0000","last_undegraded":"2024-08-17T17:33:55.252402+0000","last_fullsized":"2024-08-17T17:33:55.252402+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":413,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.286002+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:33:58.286002+0000","last_peered":"2024-08-17T17:33:58.286002+0000","last_clean":"2024-08-17T17:33:58.286002+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:33:58.286002+0000","last_undegraded":"2024-08-17T17:33:58.286002+0000","last_fullsized":"2024-08-17T17:33:58.286002+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":413,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.286021+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:33:58.286021+0000","last_peered":"2024-08-17T17:33:58.286021+0000","last_clean":"2024-08-17T17:33:58.286021+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:33:58.286021+0000","last_undegraded":"2024-08-17T17:33:58.286021+0000","last_fullsized":"2024-08-17T17:33:58.286021+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-17T17:33:55.262841+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:33:55.262841+0000","last_peered":"2024-08-17T17:33:55.262841+0000","last_clean":"2024-08-17T17:33:55.262841+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:33:55.262841+0000","last_undegraded":"2024-08-17T17:33:55.262841+0000","last_fullsized":"2024-08-17T17:33:55.262841+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":433,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.297867+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:33:58.297867+0000","last_peered":"2024-08-17T17:33:58.297867+0000","last_clean":"2024-08-17T17:33:58.297867+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:33:58.297867+0000","last_undegraded":"2024-08-17T17:33:58.297867+0000","last_fullsized":"2024-08-17T17:33:58.297867+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197443,"num_write_kb":6323585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7264,"num_bytes_recovered":456523776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":29892608,"data_stored":29884464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137243,"ondisk_log_size":137243,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201648,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462452,"kb_used_data":8044,"kb_used_omap":1,"kb_used_meta":454398,"kb_avail":93282700,"statfs":{"total":95995035648,"available":95521484800,"internally_reserved":0,"allocated":8237056,"data_stored":6775498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":465304011},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922144,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":137896,"kb_used_data":10080,"kb_used_omap":7,"kb_used_meta":127800,"kb_avail":93607256,"statfs":{"total":95995035648,"available":95853830144,"internally_reserved":0,"allocated":10321920,"data_stored":8871640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":130867894},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085668,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":90400,"kb_used_data":3352,"kb_used_omap":7,"kb_used_meta":87032,"kb_avail":93654752,"statfs":{"total":95995035648,"available":95902466048,"internally_reserved":0,"allocated":3432448,"data_stored":1986525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":89121209},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986558,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":453676,"kb_used_data":7460,"kb_used_omap":1,"kb_used_meta":446206,"kb_avail":93291476,"statfs":{"total":95995035648,"available":95530471424,"internally_reserved":0,"allocated":7639040,"data_stored":6185068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":456915398},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435449,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":564952,"kb_used_data":50128,"kb_used_omap":5,"kb_used_meta":514810,"kb_avail":93180200,"statfs":{"total":95995035648,"available":95416524800,"internally_reserved":0,"allocated":51331072,"data_stored":49892780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":527165513},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598976,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602840,"kb_used_data":3344,"kb_used_omap":7,"kb_used_meta":599480,"kb_avail":93142312,"statfs":{"total":95995035648,"available":95377727488,"internally_reserved":0,"allocated":3424256,"data_stored":1985313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":613867693},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762502,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":422352,"kb_used_data":3336,"kb_used_omap":7,"kb_used_meta":419000,"kb_avail":93322800,"statfs":{"total":95995035648,"available":95562547200,"internally_reserved":0,"allocated":3416064,"data_stored":1984443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":429056417},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926028,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":540712,"kb_used_data":17504,"kb_used_omap":7,"kb_used_meta":523128,"kb_avail":93204440,"statfs":{"total":95995035648,"available":95441346560,"internally_reserved":0,"allocated":17924096,"data_stored":16470775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":535683676},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1642496,"data_stored":1638424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":25624576,"data_stored":25624576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2625536,"data_stored":2621464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:34:19.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:34:19.334 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:19.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:19.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:19.484 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:19.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:19.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:19.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:19.732 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:19.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:19.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:19.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:19.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:19.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:20.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.203 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:20.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:20.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:20.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:20.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:20.475 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:20.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:20.598 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:20 smithi175 ceph-mon[101434]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 21531/84 objects degraded (25632.143%); 288 KiB/s, 122 objects/s recovering 2024-08-17T17:34:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:20 smithi064 ceph-mon[104516]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 21531/84 objects degraded (25632.143%); 288 KiB/s, 122 objects/s recovering 2024-08-17T17:34:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:20 smithi064 ceph-mon[109246]: pgmap v869: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 21531/84 objects degraded (25632.143%); 288 KiB/s, 122 objects/s recovering 2024-08-17T17:34:20.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:20.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:20.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:20.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:20.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:20.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:20.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:21.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:21.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:21.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:21.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:21.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:21.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.445 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.466 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:21.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:21.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:21.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:21.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:21.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:21.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:21.967 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:22.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:22.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:22.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:22.284 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:22.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:22.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:22.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:22.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:22 smithi064 ceph-mon[104516]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:22.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:22 smithi064 ceph-mon[109246]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:22.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.469 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:22.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:22.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:22.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:22 smithi175 ceph-mon[101434]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:22.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:22.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:22.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:22.761 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:22.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:22.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:22.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:22.965 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:23.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:23.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:23.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:23.214 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:23.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.371 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:34:23.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:23.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:23.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:23 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/241436981' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:23 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 21531/84 objects degraded (25632.143%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/241436981' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 21531/84 objects degraded (25632.143%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/241436981' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 21531/84 objects degraded (25632.143%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:23.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:23.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:23.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:23.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:23.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:24.087 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:24.190 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:24.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.255 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:23] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:34:24.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:24.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:24.438 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:24.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[104516]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:24.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:24.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:24.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[104516]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:34:24.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[109246]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:24.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:24.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:24.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:24 smithi064 ceph-mon[109246]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:34:24.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:24 smithi175 ceph-mon[101434]: pgmap v871: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 21531/84 objects degraded (25632.143%); 0 B/s, 107 objects/s recovering 2024-08-17T17:34:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:24 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:24 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:24 smithi175 ceph-mon[101434]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:34:24.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:24.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:24.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:24.957 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:25.082 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:25.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:25.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:25.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:25.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.452 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:25.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:25.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:25.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:25.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.944 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:26.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:26.068 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:26.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:26.191 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:26.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:26.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:26.417 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:26.438 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:26.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:26.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:26 smithi175 ceph-mon[101434]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:26.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:26.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:26 smithi064 ceph-mon[104516]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:26.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:26 smithi064 ceph-mon[109246]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:26.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:26.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:26.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:26.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:27.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:27.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:27.158 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:27.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:27.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:27.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:27.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:27.426 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:27.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:27.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:27.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:27.672 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:27.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:27.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:27.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:27.916 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:27.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:28.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:28.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:28.162 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:28.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:28.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:28.411 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:28.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:28.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:28.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:28.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:28.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:28 smithi175 ceph-mon[101434]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 20660/84 objects degraded (24595.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:28 smithi064 ceph-mon[104516]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 20660/84 objects degraded (24595.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:28 smithi064 ceph-mon[109246]: pgmap v874: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 20660/84 objects degraded (24595.238%); 0 B/s, 161 objects/s recovering 2024-08-17T17:34:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 20660/84 objects degraded (24595.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:28.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:28.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:28.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:29.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.131 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:34:29.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.524 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:29.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:29.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:29.892 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:29.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:29.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:30.017 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:30.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:30.118 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:30.139 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:30.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.241 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:30.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:30.387 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:30.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:30.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:30.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:30.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:30 smithi175 ceph-mon[101434]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:30 smithi064 ceph-mon[104516]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:30.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:30 smithi064 ceph-mon[109246]: pgmap v875: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:30.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:30.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:30.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:30.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:30.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:31.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:31.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:31.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:31.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:31.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:31.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:31.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:31.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:31.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:31.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:31.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.880 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:31.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:32.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:32.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:32.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:32.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:32.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:32.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:32.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:32.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:32.496 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:32.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:32.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:32 smithi175 ceph-mon[101434]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:32 smithi064 ceph-mon[104516]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:32 smithi064 ceph-mon[109246]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 231 objects/s recovering 2024-08-17T17:34:32.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:32.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:32.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:32.866 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:32.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:33.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:33.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:33.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:33.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:33.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:33.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:33.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:33.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:33.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:33.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:33.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:33.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:33.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:33.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:33.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:33 smithi175 ceph-mon[101434]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:34:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 19211/84 objects degraded (22870.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:33 smithi064 ceph-mon[104516]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:34:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 19211/84 objects degraded (22870.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:33.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:33 smithi064 ceph-mon[109246]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:34:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 19211/84 objects degraded (22870.238%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:33.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:33.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:33.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:33.894 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:33.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:34.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:34.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:34.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:33] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:34:34.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:34.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:34.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:34:34.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:34.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:34.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:34.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:34.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:34.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:34.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:34 smithi175 ceph-mon[101434]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 238 objects/s recovering 2024-08-17T17:34:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:34 smithi175 ceph-mon[101434]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:34:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:34 smithi064 ceph-mon[104516]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 238 objects/s recovering 2024-08-17T17:34:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:34 smithi064 ceph-mon[104516]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:34:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:34 smithi064 ceph-mon[109246]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 19211/84 objects degraded (22870.238%); 0 B/s, 238 objects/s recovering 2024-08-17T17:34:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:34 smithi064 ceph-mon[109246]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:34:34.725 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:34:34.726 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:34:34.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:34.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:34.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:34.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:34.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:35.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:35.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:35.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:35.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:35.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:35.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:35.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:35.514 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:35.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:35.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:35 smithi175 ceph-mon[101434]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:35.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:35 smithi064 ceph-mon[104516]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:35.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:35 smithi064 ceph-mon[109246]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:35.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:35.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:35.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:35.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:35.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:36.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:36.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:36.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:36.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:36.253 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:36.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:36.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:36.407 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' removed 2024-08-17T17:34:36.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:36.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:36.601 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:36.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:36.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:36 smithi064 ceph-mon[104516]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 17802/84 objects degraded (21192.857%); 0 B/s, 357 objects/s recovering 2024-08-17T17:34:36.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:36 smithi064 ceph-mon[109246]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 17802/84 objects degraded (21192.857%); 0 B/s, 357 objects/s recovering 2024-08-17T17:34:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:36.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:36.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:36.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:36.872 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:36 smithi175 ceph-mon[101434]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 17802/84 objects degraded (21192.857%); 0 B/s, 357 objects/s recovering 2024-08-17T17:34:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:36.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:36.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:37.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:37.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:37.181 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":879,"stamp":"2024-08-17T17:34:33.399026+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19211,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":198737,"num_write_kb":6338626,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10995,"num_bytes_recovered":466689859,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138596,"ondisk_log_size":138596,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3212996,"kb_used_data":40936,"kb_used_omap":46,"kb_used_meta":3171921,"kb_avail":746748220,"statfs":{"total":767960285184,"available":764670177280,"internally_reserved":0,"allocated":41918464,"data_stored":30321634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":3248047347},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1449,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1449,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.292714"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":397,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.748080+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:34:23.748080+0000","last_peered":"2024-08-17T17:34:23.748080+0000","last_clean":"2024-08-17T17:34:23.748080+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:34:23.748080+0000","last_undegraded":"2024-08-17T17:34:23.748080+0000","last_fullsized":"2024-08-17T17:34:23.748080+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":267,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293556+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:33:58.293556+0000","last_peered":"2024-08-17T17:33:58.293556+0000","last_clean":"2024-08-17T17:33:58.293556+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:33:58.293556+0000","last_undegraded":"2024-08-17T17:33:58.293556+0000","last_fullsized":"2024-08-17T17:33:58.293556+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251939+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:33:58.251939+0000","last_peered":"2024-08-17T17:33:58.251939+0000","last_clean":"2024-08-17T17:33:58.251939+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:33:58.251939+0000","last_undegraded":"2024-08-17T17:33:58.251939+0000","last_fullsized":"2024-08-17T17:33:58.251939+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":389,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368206+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:34:23.368206+0000","last_peered":"2024-08-17T17:34:23.368206+0000","last_clean":"2024-08-17T17:34:23.368206+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:34:23.368206+0000","last_undegraded":"2024-08-17T17:34:23.368206+0000","last_fullsized":"2024-08-17T17:34:23.368206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":389,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368966+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:34:23.368966+0000","last_peered":"2024-08-17T17:34:23.368966+0000","last_clean":"2024-08-17T17:34:23.368966+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:34:23.368966+0000","last_undegraded":"2024-08-17T17:34:23.368966+0000","last_fullsized":"2024-08-17T17:34:23.368966+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":253,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293507+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:33:58.293507+0000","last_peered":"2024-08-17T17:33:58.293507+0000","last_clean":"2024-08-17T17:33:58.293507+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:33:58.293507+0000","last_undegraded":"2024-08-17T17:33:58.293507+0000","last_fullsized":"2024-08-17T17:33:58.293507+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":397,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.748270+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:34:23.748270+0000","last_peered":"2024-08-17T17:34:23.748270+0000","last_clean":"2024-08-17T17:34:23.748270+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:34:23.748270+0000","last_undegraded":"2024-08-17T17:34:23.748270+0000","last_fullsized":"2024-08-17T17:34:23.748270+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":201,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.303281+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:33:58.303281+0000","last_peered":"2024-08-17T17:33:58.303281+0000","last_clean":"2024-08-17T17:33:58.303281+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:33:58.303281+0000","last_undegraded":"2024-08-17T17:33:58.303281+0000","last_fullsized":"2024-08-17T17:33:58.303281+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"269'12450","reported_seq":12554,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:23.777630+0000","last_change":"2024-08-17T17:31:37.408577+0000","last_active":"2024-08-17T17:34:23.777630+0000","last_peered":"2024-08-17T17:34:23.777630+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.396040+0000","last_became_peered":"2024-08-17T17:31:37.396040+0000","last_unstale":"2024-08-17T17:34:23.777630+0000","last_undegraded":"2024-08-17T17:31:37.194642+0000","last_fullsized":"2024-08-17T17:31:37.151313+0000","mapping_epoch":248,"log_start":"236'4500","ondisk_log_start":"236'4500","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7950,"log_dups_size":0,"ondisk_log_size":7950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12450,"num_write_kb":398400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-6063}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":262,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.294001+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:33:58.294001+0000","last_peered":"2024-08-17T17:33:58.294001+0000","last_clean":"2024-08-17T17:33:58.294001+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:33:58.294001+0000","last_undegraded":"2024-08-17T17:33:58.294001+0000","last_fullsized":"2024-08-17T17:33:58.294001+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251719+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:33:58.251719+0000","last_peered":"2024-08-17T17:33:58.251719+0000","last_clean":"2024-08-17T17:33:58.251719+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:33:58.251719+0000","last_undegraded":"2024-08-17T17:33:58.251719+0000","last_fullsized":"2024-08-17T17:33:58.251719+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"269'12274","reported_seq":12417,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747899+0000","last_change":"2024-08-17T17:31:17.798927+0000","last_active":"2024-08-17T17:34:23.747899+0000","last_peered":"2024-08-17T17:34:23.747899+0000","last_clean":"2024-08-17T17:34:23.747899+0000","last_became_active":"2024-08-17T17:31:17.603128+0000","last_became_peered":"2024-08-17T17:31:17.603128+0000","last_unstale":"2024-08-17T17:34:23.747899+0000","last_undegraded":"2024-08-17T17:34:23.747899+0000","last_fullsized":"2024-08-17T17:34:23.747899+0000","mapping_epoch":242,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:37.753356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12274,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":983040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":263,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293861+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:33:58.293861+0000","last_peered":"2024-08-17T17:33:58.293861+0000","last_clean":"2024-08-17T17:33:58.293861+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:33:58.293861+0000","last_undegraded":"2024-08-17T17:33:58.293861+0000","last_fullsized":"2024-08-17T17:33:58.293861+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284503+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:33:58.284503+0000","last_peered":"2024-08-17T17:33:58.284503+0000","last_clean":"2024-08-17T17:33:58.284503+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:33:58.284503+0000","last_undegraded":"2024-08-17T17:33:58.284503+0000","last_fullsized":"2024-08-17T17:33:58.284503+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"269'12532","reported_seq":12629,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368263+0000","last_change":"2024-08-17T17:29:57.202283+0000","last_active":"2024-08-17T17:34:23.368263+0000","last_peered":"2024-08-17T17:34:23.368263+0000","last_clean":"2024-08-17T17:34:23.368263+0000","last_became_active":"2024-08-17T17:29:57.201797+0000","last_became_peered":"2024-08-17T17:29:57.201797+0000","last_unstale":"2024-08-17T17:34:23.368263+0000","last_undegraded":"2024-08-17T17:34:23.368263+0000","last_fullsized":"2024-08-17T17:34:23.368263+0000","mapping_epoch":228,"log_start":"235'3700","ondisk_log_start":"235'3700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8832,"log_dups_size":0,"ondisk_log_size":8832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:31.699096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12532,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":280,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293899+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:33:58.293899+0000","last_peered":"2024-08-17T17:33:58.293899+0000","last_clean":"2024-08-17T17:33:58.293899+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:33:58.293899+0000","last_undegraded":"2024-08-17T17:33:58.293899+0000","last_fullsized":"2024-08-17T17:33:58.293899+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":181,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.303397+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:33:58.303397+0000","last_peered":"2024-08-17T17:33:58.303397+0000","last_clean":"2024-08-17T17:33:58.303397+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:33:58.303397+0000","last_undegraded":"2024-08-17T17:33:58.303397+0000","last_fullsized":"2024-08-17T17:33:58.303397+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"269'12310","reported_seq":12408,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.774888+0000","last_change":"2024-08-17T17:29:57.200346+0000","last_active":"2024-08-17T17:34:23.774888+0000","last_peered":"2024-08-17T17:34:23.774888+0000","last_clean":"2024-08-17T17:34:23.774888+0000","last_became_active":"2024-08-17T17:29:57.200066+0000","last_became_peered":"2024-08-17T17:29:57.200066+0000","last_unstale":"2024-08-17T17:34:23.774888+0000","last_undegraded":"2024-08-17T17:34:23.774888+0000","last_fullsized":"2024-08-17T17:34:23.774888+0000","mapping_epoch":228,"log_start":"234'2300","ondisk_log_start":"234'2300","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:31.030646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12310,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251662+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:33:58.251662+0000","last_peered":"2024-08-17T17:33:58.251662+0000","last_clean":"2024-08-17T17:33:58.251662+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:33:58.251662+0000","last_undegraded":"2024-08-17T17:33:58.251662+0000","last_fullsized":"2024-08-17T17:33:58.251662+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":403,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368560+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:34:23.368560+0000","last_peered":"2024-08-17T17:34:23.368560+0000","last_clean":"2024-08-17T17:34:23.368560+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:34:23.368560+0000","last_undegraded":"2024-08-17T17:34:23.368560+0000","last_fullsized":"2024-08-17T17:34:23.368560+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"269'12378","reported_seq":12475,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368428+0000","last_change":"2024-08-17T17:29:57.204892+0000","last_active":"2024-08-17T17:34:23.368428+0000","last_peered":"2024-08-17T17:34:23.368428+0000","last_clean":"2024-08-17T17:34:23.368428+0000","last_became_active":"2024-08-17T17:29:57.204591+0000","last_became_peered":"2024-08-17T17:29:57.204591+0000","last_unstale":"2024-08-17T17:34:23.368428+0000","last_undegraded":"2024-08-17T17:34:23.368428+0000","last_fullsized":"2024-08-17T17:34:23.368428+0000","mapping_epoch":228,"log_start":"234'3500","ondisk_log_start":"234'3500","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8878,"log_dups_size":0,"ondisk_log_size":8878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:01:58.087057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12378,"num_write_kb":396096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":418,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.748050+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:34:23.748050+0000","last_peered":"2024-08-17T17:34:23.748050+0000","last_clean":"2024-08-17T17:34:23.748050+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:34:23.748050+0000","last_undegraded":"2024-08-17T17:34:23.748050+0000","last_fullsized":"2024-08-17T17:34:23.748050+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284556+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:33:58.284556+0000","last_peered":"2024-08-17T17:33:58.284556+0000","last_clean":"2024-08-17T17:33:58.284556+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:33:58.284556+0000","last_undegraded":"2024-08-17T17:33:58.284556+0000","last_fullsized":"2024-08-17T17:33:58.284556+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"269'12506","reported_seq":12646,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775360+0000","last_change":"2024-08-17T17:31:17.661359+0000","last_active":"2024-08-17T17:34:23.775360+0000","last_peered":"2024-08-17T17:34:23.775360+0000","last_clean":"2024-08-17T17:34:23.775360+0000","last_became_active":"2024-08-17T17:31:17.599056+0000","last_became_peered":"2024-08-17T17:31:17.599056+0000","last_unstale":"2024-08-17T17:34:23.775360+0000","last_undegraded":"2024-08-17T17:34:23.775360+0000","last_fullsized":"2024-08-17T17:34:23.775360+0000","mapping_epoch":242,"log_start":"234'2500","ondisk_log_start":"234'2500","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:12.648106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12506,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":188,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:06.776767+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:34:06.776767+0000","last_peered":"2024-08-17T17:34:06.776767+0000","last_clean":"2024-08-17T17:34:06.776767+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:34:06.776767+0000","last_undegraded":"2024-08-17T17:34:06.776767+0000","last_fullsized":"2024-08-17T17:34:06.776767+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":158,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284705+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:33:58.284705+0000","last_peered":"2024-08-17T17:33:58.284705+0000","last_clean":"2024-08-17T17:33:58.284705+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:33:58.284705+0000","last_undegraded":"2024-08-17T17:33:58.284705+0000","last_fullsized":"2024-08-17T17:33:58.284705+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"269'12360","reported_seq":12458,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:23.777711+0000","last_change":"2024-08-17T17:31:37.408160+0000","last_active":"2024-08-17T17:34:23.777711+0000","last_peered":"2024-08-17T17:34:23.777711+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.395243+0000","last_became_peered":"2024-08-17T17:31:37.395243+0000","last_unstale":"2024-08-17T17:34:23.777711+0000","last_undegraded":"2024-08-17T17:31:37.192943+0000","last_fullsized":"2024-08-17T17:31:37.149514+0000","mapping_epoch":248,"log_start":"235'4400","ondisk_log_start":"235'4400","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12360,"num_write_kb":395457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":-6013}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284415+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:33:58.284415+0000","last_peered":"2024-08-17T17:33:58.284415+0000","last_clean":"2024-08-17T17:33:58.284415+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:33:58.284415+0000","last_undegraded":"2024-08-17T17:33:58.284415+0000","last_fullsized":"2024-08-17T17:33:58.284415+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284753+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:33:58.284753+0000","last_peered":"2024-08-17T17:33:58.284753+0000","last_clean":"2024-08-17T17:33:58.284753+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:33:58.284753+0000","last_undegraded":"2024-08-17T17:33:58.284753+0000","last_fullsized":"2024-08-17T17:33:58.284753+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"269'12298","reported_seq":22214,"reported_epoch":270,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:28.613123+0000","last_change":"2024-08-17T17:33:21.287503+0000","last_active":"2024-08-17T17:34:28.613123+0000","last_peered":"2024-08-17T17:34:28.613123+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.358516+0000","last_became_peered":"2024-08-17T17:31:37.358516+0000","last_unstale":"2024-08-17T17:34:28.613123+0000","last_undegraded":"2024-08-17T17:31:37.192795+0000","last_fullsized":"2024-08-17T17:31:37.150416+0000","mapping_epoch":248,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1096,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12298,"num_write_kb":393536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4910,"num_bytes_recovered":77201408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1096}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":399,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747352+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:34:23.747352+0000","last_peered":"2024-08-17T17:34:23.747352+0000","last_clean":"2024-08-17T17:34:23.747352+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:34:23.747352+0000","last_undegraded":"2024-08-17T17:34:23.747352+0000","last_fullsized":"2024-08-17T17:34:23.747352+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":387,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775161+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:34:23.775161+0000","last_peered":"2024-08-17T17:34:23.775161+0000","last_clean":"2024-08-17T17:34:23.775161+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:34:23.775161+0000","last_undegraded":"2024-08-17T17:34:23.775161+0000","last_fullsized":"2024-08-17T17:34:23.775161+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"269'12236","reported_seq":12333,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368761+0000","last_change":"2024-08-17T17:29:57.204368+0000","last_active":"2024-08-17T17:34:23.368761+0000","last_peered":"2024-08-17T17:34:23.368761+0000","last_clean":"2024-08-17T17:34:23.368761+0000","last_became_active":"2024-08-17T17:29:57.204070+0000","last_became_peered":"2024-08-17T17:29:57.204070+0000","last_unstale":"2024-08-17T17:34:23.368761+0000","last_undegraded":"2024-08-17T17:34:23.368761+0000","last_fullsized":"2024-08-17T17:34:23.368761+0000","mapping_epoch":228,"log_start":"234'3400","ondisk_log_start":"234'3400","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8836,"log_dups_size":0,"ondisk_log_size":8836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.001063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12236,"num_write_kb":391552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":583,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:01.178242+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:34:01.178242+0000","last_peered":"2024-08-17T17:34:01.178242+0000","last_clean":"2024-08-17T17:34:01.178242+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:34:01.178242+0000","last_undegraded":"2024-08-17T17:34:01.178242+0000","last_fullsized":"2024-08-17T17:34:01.178242+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":859,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.252271+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:33:58.252271+0000","last_peered":"2024-08-17T17:33:58.252271+0000","last_clean":"2024-08-17T17:33:58.252271+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:33:58.252271+0000","last_undegraded":"2024-08-17T17:33:58.252271+0000","last_fullsized":"2024-08-17T17:33:58.252271+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":382,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747964+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:34:23.747964+0000","last_peered":"2024-08-17T17:34:23.747964+0000","last_clean":"2024-08-17T17:34:23.747964+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:34:23.747964+0000","last_undegraded":"2024-08-17T17:34:23.747964+0000","last_fullsized":"2024-08-17T17:34:23.747964+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":172,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284637+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:33:58.284637+0000","last_peered":"2024-08-17T17:33:58.284637+0000","last_clean":"2024-08-17T17:33:58.284637+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:33:58.284637+0000","last_undegraded":"2024-08-17T17:33:58.284637+0000","last_fullsized":"2024-08-17T17:33:58.284637+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"269'12350","reported_seq":12402,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:11.192630+0000","last_change":"2024-08-17T17:29:57.203033+0000","last_active":"2024-08-17T17:34:11.192630+0000","last_peered":"2024-08-17T17:34:11.192630+0000","last_clean":"2024-08-17T17:34:11.192630+0000","last_became_active":"2024-08-17T17:29:57.202766+0000","last_became_peered":"2024-08-17T17:29:57.202766+0000","last_unstale":"2024-08-17T17:34:11.192630+0000","last_undegraded":"2024-08-17T17:34:11.192630+0000","last_fullsized":"2024-08-17T17:34:11.192630+0000","mapping_epoch":228,"log_start":"235'3700","ondisk_log_start":"235'3700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8650,"log_dups_size":0,"ondisk_log_size":8650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:06.312698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12350,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":645,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293661+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:33:58.293661+0000","last_peered":"2024-08-17T17:33:58.293661+0000","last_clean":"2024-08-17T17:33:58.293661+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:33:58.293661+0000","last_undegraded":"2024-08-17T17:33:58.293661+0000","last_fullsized":"2024-08-17T17:33:58.293661+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":404,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368372+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:34:23.368372+0000","last_peered":"2024-08-17T17:34:23.368372+0000","last_clean":"2024-08-17T17:34:23.368372+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:34:23.368372+0000","last_undegraded":"2024-08-17T17:34:23.368372+0000","last_fullsized":"2024-08-17T17:34:23.368372+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":400,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.369048+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:34:23.369048+0000","last_peered":"2024-08-17T17:34:23.369048+0000","last_clean":"2024-08-17T17:34:23.369048+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:34:23.369048+0000","last_undegraded":"2024-08-17T17:34:23.369048+0000","last_fullsized":"2024-08-17T17:34:23.369048+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"269'12268","reported_seq":12365,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368478+0000","last_change":"2024-08-17T17:29:57.202172+0000","last_active":"2024-08-17T17:34:23.368478+0000","last_peered":"2024-08-17T17:34:23.368478+0000","last_clean":"2024-08-17T17:34:23.368478+0000","last_became_active":"2024-08-17T17:29:57.201785+0000","last_became_peered":"2024-08-17T17:29:57.201785+0000","last_unstale":"2024-08-17T17:34:23.368478+0000","last_undegraded":"2024-08-17T17:34:23.368478+0000","last_fullsized":"2024-08-17T17:34:23.368478+0000","mapping_epoch":228,"log_start":"234'3400","ondisk_log_start":"234'3400","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:49.084764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12268,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":801,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.774803+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:34:23.774803+0000","last_peered":"2024-08-17T17:34:23.774803+0000","last_clean":"2024-08-17T17:34:23.774803+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:34:23.774803+0000","last_undegraded":"2024-08-17T17:34:23.774803+0000","last_fullsized":"2024-08-17T17:34:23.774803+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":393,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775143+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:34:23.775143+0000","last_peered":"2024-08-17T17:34:23.775143+0000","last_clean":"2024-08-17T17:34:23.775143+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:34:23.775143+0000","last_undegraded":"2024-08-17T17:34:23.775143+0000","last_fullsized":"2024-08-17T17:34:23.775143+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":440,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.776050+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:34:23.776050+0000","last_peered":"2024-08-17T17:34:23.776050+0000","last_clean":"2024-08-17T17:34:23.776050+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:34:23.776050+0000","last_undegraded":"2024-08-17T17:34:23.776050+0000","last_fullsized":"2024-08-17T17:34:23.776050+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"269'12382","reported_seq":12477,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:34:23.368501+0000","last_change":"2024-08-17T17:31:37.399805+0000","last_active":"2024-08-17T17:34:23.368501+0000","last_peered":"2024-08-17T17:34:23.368501+0000","last_clean":"2024-08-17T17:29:56.176182+0000","last_became_active":"2024-08-17T17:31:37.386849+0000","last_became_peered":"2024-08-17T17:31:37.386849+0000","last_unstale":"2024-08-17T17:34:23.368501+0000","last_undegraded":"2024-08-17T17:31:37.193155+0000","last_fullsized":"2024-08-17T17:31:37.153842+0000","mapping_epoch":248,"log_start":"234'3500","ondisk_log_start":"234'3500","created":228,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6039,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12382,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-6039}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":982,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.252063+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:33:58.252063+0000","last_peered":"2024-08-17T17:33:58.252063+0000","last_clean":"2024-08-17T17:33:58.252063+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:33:58.252063+0000","last_undegraded":"2024-08-17T17:33:58.252063+0000","last_fullsized":"2024-08-17T17:33:58.252063+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":218,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775114+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:34:23.775114+0000","last_peered":"2024-08-17T17:34:23.775114+0000","last_clean":"2024-08-17T17:34:23.775114+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:34:23.775114+0000","last_undegraded":"2024-08-17T17:34:23.775114+0000","last_fullsized":"2024-08-17T17:34:23.775114+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":287,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775993+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:34:23.775993+0000","last_peered":"2024-08-17T17:34:23.775993+0000","last_clean":"2024-08-17T17:34:23.775993+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:34:23.775993+0000","last_undegraded":"2024-08-17T17:34:23.775993+0000","last_fullsized":"2024-08-17T17:34:23.775993+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"269'12206","reported_seq":24139,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.648617+0000","last_change":"2024-08-17T17:33:22.708397+0000","last_active":"2024-08-17T17:34:09.648617+0000","last_peered":"2024-08-17T17:34:09.648617+0000","last_clean":"2024-08-17T17:34:09.648617+0000","last_became_active":"2024-08-17T17:33:22.677716+0000","last_became_peered":"2024-08-17T17:33:22.677716+0000","last_unstale":"2024-08-17T17:34:09.648617+0000","last_undegraded":"2024-08-17T17:34:09.648617+0000","last_fullsized":"2024-08-17T17:34:09.648617+0000","mapping_epoch":263,"log_start":"235'4000","ondisk_log_start":"235'4000","created":228,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:23.405360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5932,"num_bytes_recovered":376832000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":51,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251795+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:33:58.251795+0000","last_peered":"2024-08-17T17:33:58.251795+0000","last_clean":"2024-08-17T17:33:58.251795+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:33:58.251795+0000","last_undegraded":"2024-08-17T17:33:58.251795+0000","last_fullsized":"2024-08-17T17:33:58.251795+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":686,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.776228+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:34:23.776228+0000","last_peered":"2024-08-17T17:34:23.776228+0000","last_clean":"2024-08-17T17:34:23.776228+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:34:23.776228+0000","last_undegraded":"2024-08-17T17:34:23.776228+0000","last_fullsized":"2024-08-17T17:34:23.776228+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":410,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775634+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:34:23.775634+0000","last_peered":"2024-08-17T17:34:23.775634+0000","last_clean":"2024-08-17T17:34:23.775634+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:34:23.775634+0000","last_undegraded":"2024-08-17T17:34:23.775634+0000","last_fullsized":"2024-08-17T17:34:23.775634+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"269'12426","reported_seq":12492,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:09.631220+0000","last_change":"2024-08-17T17:31:17.646602+0000","last_active":"2024-08-17T17:34:09.631220+0000","last_peered":"2024-08-17T17:34:09.631220+0000","last_clean":"2024-08-17T17:34:09.631220+0000","last_became_active":"2024-08-17T17:31:17.591931+0000","last_became_peered":"2024-08-17T17:31:17.591931+0000","last_unstale":"2024-08-17T17:34:09.631220+0000","last_undegraded":"2024-08-17T17:34:09.631220+0000","last_fullsized":"2024-08-17T17:34:09.631220+0000","mapping_epoch":242,"log_start":"234'2400","ondisk_log_start":"234'2400","created":228,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:12:29.536081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12426,"num_write_kb":397632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":407,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.776131+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:34:23.776131+0000","last_peered":"2024-08-17T17:34:23.776131+0000","last_clean":"2024-08-17T17:34:23.776131+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:34:23.776131+0000","last_undegraded":"2024-08-17T17:34:23.776131+0000","last_fullsized":"2024-08-17T17:34:23.776131+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":835,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293889+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:33:58.293889+0000","last_peered":"2024-08-17T17:33:58.293889+0000","last_clean":"2024-08-17T17:33:58.293889+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:33:58.293889+0000","last_undegraded":"2024-08-17T17:33:58.293889+0000","last_fullsized":"2024-08-17T17:33:58.293889+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":404,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368462+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:34:23.368462+0000","last_peered":"2024-08-17T17:34:23.368462+0000","last_clean":"2024-08-17T17:34:23.368462+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:34:23.368462+0000","last_undegraded":"2024-08-17T17:34:23.368462+0000","last_fullsized":"2024-08-17T17:34:23.368462+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"269'12280","reported_seq":12377,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747978+0000","last_change":"2024-08-17T17:29:57.198724+0000","last_active":"2024-08-17T17:34:23.747978+0000","last_peered":"2024-08-17T17:34:23.747978+0000","last_clean":"2024-08-17T17:34:23.747978+0000","last_became_active":"2024-08-17T17:29:57.198432+0000","last_became_peered":"2024-08-17T17:29:57.198432+0000","last_unstale":"2024-08-17T17:34:23.747978+0000","last_undegraded":"2024-08-17T17:34:23.747978+0000","last_fullsized":"2024-08-17T17:34:23.747978+0000","mapping_epoch":228,"log_start":"236'4700","ondisk_log_start":"236'4700","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:23:54.902580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12280,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":397,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747554+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:34:23.747554+0000","last_peered":"2024-08-17T17:34:23.747554+0000","last_clean":"2024-08-17T17:34:23.747554+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:34:23.747554+0000","last_undegraded":"2024-08-17T17:34:23.747554+0000","last_fullsized":"2024-08-17T17:34:23.747554+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":916,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.776058+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:34:23.776058+0000","last_peered":"2024-08-17T17:34:23.776058+0000","last_clean":"2024-08-17T17:34:23.776058+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:34:23.776058+0000","last_undegraded":"2024-08-17T17:34:23.776058+0000","last_fullsized":"2024-08-17T17:34:23.776058+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":394,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747504+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:34:23.747504+0000","last_peered":"2024-08-17T17:34:23.747504+0000","last_clean":"2024-08-17T17:34:23.747504+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:34:23.747504+0000","last_undegraded":"2024-08-17T17:34:23.747504+0000","last_fullsized":"2024-08-17T17:34:23.747504+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"269'12358","reported_seq":12455,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747931+0000","last_change":"2024-08-17T17:29:57.199611+0000","last_active":"2024-08-17T17:34:23.747931+0000","last_peered":"2024-08-17T17:34:23.747931+0000","last_clean":"2024-08-17T17:34:23.747931+0000","last_became_active":"2024-08-17T17:29:57.199174+0000","last_became_peered":"2024-08-17T17:29:57.199174+0000","last_unstale":"2024-08-17T17:34:23.747931+0000","last_undegraded":"2024-08-17T17:34:23.747931+0000","last_fullsized":"2024-08-17T17:34:23.747931+0000","mapping_epoch":228,"log_start":"236'4800","ondisk_log_start":"236'4800","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:29:56.176182+0000","last_clean_scrub_stamp":"2024-08-17T17:29:56.176182+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:35.587097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12358,"num_write_kb":395456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":415,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368858+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:34:23.368858+0000","last_peered":"2024-08-17T17:34:23.368858+0000","last_clean":"2024-08-17T17:34:23.368858+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:34:23.368858+0000","last_undegraded":"2024-08-17T17:34:23.368858+0000","last_fullsized":"2024-08-17T17:34:23.368858+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":391,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:06.192220+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:34:06.192220+0000","last_peered":"2024-08-17T17:34:06.192220+0000","last_clean":"2024-08-17T17:34:06.192220+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:34:06.192220+0000","last_undegraded":"2024-08-17T17:34:06.192220+0000","last_fullsized":"2024-08-17T17:34:06.192220+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":209,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284547+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:33:58.284547+0000","last_peered":"2024-08-17T17:33:58.284547+0000","last_clean":"2024-08-17T17:33:58.284547+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:33:58.284547+0000","last_undegraded":"2024-08-17T17:33:58.284547+0000","last_fullsized":"2024-08-17T17:33:58.284547+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:01.776327+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:34:01.776327+0000","last_peered":"2024-08-17T17:34:01.776327+0000","last_clean":"2024-08-17T17:34:01.776327+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:34:01.776327+0000","last_undegraded":"2024-08-17T17:34:01.776327+0000","last_fullsized":"2024-08-17T17:34:01.776327+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":183,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:34:01.776348+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:34:01.776348+0000","last_peered":"2024-08-17T17:34:01.776348+0000","last_clean":"2024-08-17T17:34:01.776348+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:34:01.776348+0000","last_undegraded":"2024-08-17T17:34:01.776348+0000","last_fullsized":"2024-08-17T17:34:01.776348+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":392,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775435+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:34:23.775435+0000","last_peered":"2024-08-17T17:34:23.775435+0000","last_clean":"2024-08-17T17:34:23.775435+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:34:23.775435+0000","last_undegraded":"2024-08-17T17:34:23.775435+0000","last_fullsized":"2024-08-17T17:34:23.775435+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":267,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.293788+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:33:58.293788+0000","last_peered":"2024-08-17T17:33:58.293788+0000","last_clean":"2024-08-17T17:33:58.293788+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:33:58.293788+0000","last_undegraded":"2024-08-17T17:33:58.293788+0000","last_fullsized":"2024-08-17T17:33:58.293788+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":51,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251462+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:33:58.251462+0000","last_peered":"2024-08-17T17:33:58.251462+0000","last_clean":"2024-08-17T17:33:58.251462+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:33:58.251462+0000","last_undegraded":"2024-08-17T17:33:58.251462+0000","last_fullsized":"2024-08-17T17:33:58.251462+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":359,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747611+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:34:23.747611+0000","last_peered":"2024-08-17T17:34:23.747611+0000","last_clean":"2024-08-17T17:34:23.747611+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:34:23.747611+0000","last_undegraded":"2024-08-17T17:34:23.747611+0000","last_fullsized":"2024-08-17T17:34:23.747611+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":418,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368782+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:34:23.368782+0000","last_peered":"2024-08-17T17:34:23.368782+0000","last_clean":"2024-08-17T17:34:23.368782+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:34:23.368782+0000","last_undegraded":"2024-08-17T17:34:23.368782+0000","last_fullsized":"2024-08-17T17:34:23.368782+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":408,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747681+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:34:23.747681+0000","last_peered":"2024-08-17T17:34:23.747681+0000","last_clean":"2024-08-17T17:34:23.747681+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:34:23.747681+0000","last_undegraded":"2024-08-17T17:34:23.747681+0000","last_fullsized":"2024-08-17T17:34:23.747681+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.252116+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:33:58.252116+0000","last_peered":"2024-08-17T17:33:58.252116+0000","last_clean":"2024-08-17T17:33:58.252116+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:33:58.252116+0000","last_undegraded":"2024-08-17T17:33:58.252116+0000","last_fullsized":"2024-08-17T17:33:58.252116+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":413,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747354+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:34:23.747354+0000","last_peered":"2024-08-17T17:34:23.747354+0000","last_clean":"2024-08-17T17:34:23.747354+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:34:23.747354+0000","last_undegraded":"2024-08-17T17:34:23.747354+0000","last_fullsized":"2024-08-17T17:34:23.747354+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":390,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.369041+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:34:23.369041+0000","last_peered":"2024-08-17T17:34:23.369041+0000","last_clean":"2024-08-17T17:34:23.369041+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:34:23.369041+0000","last_undegraded":"2024-08-17T17:34:23.369041+0000","last_fullsized":"2024-08-17T17:34:23.369041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":394,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775207+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:34:23.775207+0000","last_peered":"2024-08-17T17:34:23.775207+0000","last_clean":"2024-08-17T17:34:23.775207+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:34:23.775207+0000","last_undegraded":"2024-08-17T17:34:23.775207+0000","last_fullsized":"2024-08-17T17:34:23.775207+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":410,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.748097+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:34:23.748097+0000","last_peered":"2024-08-17T17:34:23.748097+0000","last_clean":"2024-08-17T17:34:23.748097+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:34:23.748097+0000","last_undegraded":"2024-08-17T17:34:23.748097+0000","last_fullsized":"2024-08-17T17:34:23.748097+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":157,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284480+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:33:58.284480+0000","last_peered":"2024-08-17T17:33:58.284480+0000","last_clean":"2024-08-17T17:33:58.284480+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:33:58.284480+0000","last_undegraded":"2024-08-17T17:33:58.284480+0000","last_fullsized":"2024-08-17T17:33:58.284480+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":419,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.775158+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:34:23.775158+0000","last_peered":"2024-08-17T17:34:23.775158+0000","last_clean":"2024-08-17T17:34:23.775158+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:34:23.775158+0000","last_undegraded":"2024-08-17T17:34:23.775158+0000","last_fullsized":"2024-08-17T17:34:23.775158+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":395,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368455+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:34:23.368455+0000","last_peered":"2024-08-17T17:34:23.368455+0000","last_clean":"2024-08-17T17:34:23.368455+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:34:23.368455+0000","last_undegraded":"2024-08-17T17:34:23.368455+0000","last_fullsized":"2024-08-17T17:34:23.368455+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.285066+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:33:58.285066+0000","last_peered":"2024-08-17T17:33:58.285066+0000","last_clean":"2024-08-17T17:33:58.285066+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:33:58.285066+0000","last_undegraded":"2024-08-17T17:33:58.285066+0000","last_fullsized":"2024-08-17T17:33:58.285066+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":55,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251546+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:33:58.251546+0000","last_peered":"2024-08-17T17:33:58.251546+0000","last_clean":"2024-08-17T17:33:58.251546+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:33:58.251546+0000","last_undegraded":"2024-08-17T17:33:58.251546+0000","last_fullsized":"2024-08-17T17:33:58.251546+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":157,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.284416+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:33:58.284416+0000","last_peered":"2024-08-17T17:33:58.284416+0000","last_clean":"2024-08-17T17:33:58.284416+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:33:58.284416+0000","last_undegraded":"2024-08-17T17:33:58.284416+0000","last_fullsized":"2024-08-17T17:33:58.284416+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":65,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-17T17:33:58.251872+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:33:58.251872+0000","last_peered":"2024-08-17T17:33:58.251872+0000","last_clean":"2024-08-17T17:33:58.251872+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:33:58.251872+0000","last_undegraded":"2024-08-17T17:33:58.251872+0000","last_fullsized":"2024-08-17T17:33:58.251872+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":415,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.747358+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:34:23.747358+0000","last_peered":"2024-08-17T17:34:23.747358+0000","last_clean":"2024-08-17T17:34:23.747358+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:34:23.747358+0000","last_undegraded":"2024-08-17T17:34:23.747358+0000","last_fullsized":"2024-08-17T17:34:23.747358+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":415,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.748219+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:34:23.748219+0000","last_peered":"2024-08-17T17:34:23.748219+0000","last_clean":"2024-08-17T17:34:23.748219+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:34:23.748219+0000","last_undegraded":"2024-08-17T17:34:23.748219+0000","last_fullsized":"2024-08-17T17:34:23.748219+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.774858+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:34:23.774858+0000","last_peered":"2024-08-17T17:34:23.774858+0000","last_clean":"2024-08-17T17:34:23.774858+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:34:23.774858+0000","last_undegraded":"2024-08-17T17:34:23.774858+0000","last_fullsized":"2024-08-17T17:34:23.774858+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":435,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-17T17:34:23.368680+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:34:23.368680+0000","last_peered":"2024-08-17T17:34:23.368680+0000","last_clean":"2024-08-17T17:34:23.368680+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:34:23.368680+0000","last_undegraded":"2024-08-17T17:34:23.368680+0000","last_fullsized":"2024-08-17T17:34:23.368680+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19211,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197614,"num_write_kb":6323585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10880,"num_bytes_recovered":456523776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137414,"ondisk_log_size":137414,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201652,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462468,"kb_used_data":8056,"kb_used_omap":1,"kb_used_meta":454398,"kb_avail":93282684,"statfs":{"total":95995035648,"available":95521468416,"internally_reserved":0,"allocated":8249344,"data_stored":6783903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":465304011},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922148,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":127664,"kb_used_data":3944,"kb_used_omap":7,"kb_used_meta":123704,"kb_avail":93617488,"statfs":{"total":95995035648,"available":95864307712,"internally_reserved":0,"allocated":4038656,"data_stored":2588565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":126673590},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085672,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":90416,"kb_used_data":3364,"kb_used_omap":7,"kb_used_meta":87032,"kb_avail":93654736,"statfs":{"total":95995035648,"available":95902449664,"internally_reserved":0,"allocated":3444736,"data_stored":1994930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":89121209},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986562,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":453692,"kb_used_data":7472,"kb_used_omap":1,"kb_used_meta":446206,"kb_avail":93291460,"statfs":{"total":95995035648,"available":95530455040,"internally_reserved":0,"allocated":7651328,"data_stored":6193473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":456915398},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435453,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522344,"kb_used_data":3356,"kb_used_omap":5,"kb_used_meta":518970,"kb_avail":93222808,"statfs":{"total":95995035648,"available":95460155392,"internally_reserved":0,"allocated":3436544,"data_stored":1994369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":531425353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598980,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602856,"kb_used_data":3356,"kb_used_omap":7,"kb_used_meta":599480,"kb_avail":93142296,"statfs":{"total":95995035648,"available":95377711104,"internally_reserved":0,"allocated":3436544,"data_stored":1993718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":613867693},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762506,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":422368,"kb_used_data":3348,"kb_used_omap":7,"kb_used_meta":419000,"kb_avail":93322784,"statfs":{"total":95995035648,"available":95562530816,"internally_reserved":0,"allocated":3428352,"data_stored":1992848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":429056417},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926032,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":531188,"kb_used_data":8040,"kb_used_omap":7,"kb_used_meta":523128,"kb_avail":93213964,"statfs":{"total":95995035648,"available":95451099136,"internally_reserved":0,"allocated":8232960,"data_stored":6779828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":535683676},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:34:37.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:34:37.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:37.242 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:37.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.368 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:37.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:37.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.662 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:37.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:37 smithi064 ceph-mon[104516]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:34:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:37 smithi064 ceph-mon[109246]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:34:37.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:37.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:37.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:37.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/916556358' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:37.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:37 smithi175 ceph-mon[101434]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:34:37.913 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:37.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:34:38.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:38.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:38.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:38.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:38.285 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:38.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:38.409 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:38.496 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:34:38.496 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:34:38.497 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-08-17T17:34:38.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-08-17T17:34:38.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:38.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:38.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:34:38.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:34:38.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:34:38.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:34:38.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:38.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:38.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[104516]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17802/84 objects degraded (21192.857%), 3 pgs degraded, 3 pgs undersized) 2024-08-17T17:34:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/345284158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[109246]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17802/84 objects degraded (21192.857%), 3 pgs degraded, 3 pgs undersized) 2024-08-17T17:34:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:38 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/345284158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:38.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:38.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:38.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:38.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:38 smithi175 ceph-mon[101434]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:38 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17802/84 objects degraded (21192.857%), 3 pgs degraded, 3 pgs undersized) 2024-08-17T17:34:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:38 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/345284158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:38.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.936 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:39.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:39.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:39.162 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:39.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:39.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:39.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:39.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:39.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:39.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:39.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:39.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:39.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:39.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:39.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:39.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:39.938 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:39.999 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:40.001 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:40.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.062 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:40.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:40.291 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:40.316 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:40.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:40.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:40.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:40.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:40.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[104516]: pgmap v883: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[109246]: pgmap v883: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:40.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:40.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:40.820 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:40 smithi175 ceph-mon[101434]: pgmap v883: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:40.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:40.948 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:40.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.050 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:41.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:41.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:41.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:34:41.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:41.325 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:41.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:41.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:41.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:41.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:41.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:41.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:41.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:41.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:41.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:42.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:42.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:42.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:42.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:42.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:42.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:42 smithi064 ceph-mon[104516]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:42.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:42 smithi064 ceph-mon[109246]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:42.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:42.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:42.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:42.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:42.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.837 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:42 smithi175 ceph-mon[101434]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:42.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:42.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:43.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:43.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:43.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:43.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:43.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:43.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:43.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:43.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:43.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.963 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:43.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:44.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.118 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.247 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:44.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:44.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:44.427 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:43] "GET /metrics HTTP/1.1" 200 34287 "" "Prometheus/2.51.0" 2024-08-17T17:34:44.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:44.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:44.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:44 smithi064 ceph-mon[104516]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:44.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:44 smithi064 ceph-mon[109246]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:44.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:44.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:44.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:44 smithi175 ceph-mon[101434]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:44.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:45.018 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:45.120 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:45.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:45.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:45.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:45.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:45.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:45.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:45.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:45.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:45.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:45.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:45.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:45.914 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:45.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:46.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:46.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:46.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:46.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:46.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:46.293 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:46.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:46.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.451 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:46.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:46.499 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:46 smithi064 ceph-mon[104516]: pgmap v886: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:46.499 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:46 smithi064 ceph-mon[109246]: pgmap v886: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:46.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:46.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:46.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:46.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:46.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:46.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:46.824 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:46 smithi175 ceph-mon[101434]: pgmap v886: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:46.926 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:46.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:47.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:47.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:47.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:47.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:47.298 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:34:47.298 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71399 6.13 6:ce72d6cc:::benchmark_data_smithi064_185391_object71398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11625709100002, 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.186731456, 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71399 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147927+0000", 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9200000000000001e-07 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147969+0000", 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1732e-05 2024-08-17T17:34:47.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147976+0000", 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147981+0000", 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150092+0000", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021112190000000001 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150098+0000", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9739999999999999e-06 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314529+0000", 2024-08-17T17:34:47.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1644304400000001 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314695+0000", 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016647400000000001 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.314863+0000", 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016801899999999999 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318404+0000", 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035409460000000001 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321548+0000", 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031436210000000001 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326324+0000", 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047760069999999996 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332911+0000", 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065868460000000004 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332962+0000", 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1026000000000002e-05 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334658+0000", 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016963169999999999 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71427 6.13 6:c8d80d54:::benchmark_data_smithi064_185391_object71426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.116195839, 2024-08-17T17:34:47.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1881262239999999, 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71427 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.147988+0000", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9699999999999996e-07 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148026+0000", 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7808000000000001e-05 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148033+0000", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5409999999999997e-06 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148038+0000", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7400000000000001e-06 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150120+0000", 2024-08-17T17:34:47.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020819060000000001 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150126+0000", 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4450000000000004e-06 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316489+0000", 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1663628049999999 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316645+0000", 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015632600000000001 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.316822+0000", 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017728700000000001 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318448+0000", 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001625472 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321553+0000", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031055340000000001 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326371+0000", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004817867 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334787+0000", 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084156920000000007 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334912+0000", 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336114+0000", 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012023540000000001 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.314 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71428 6.13 6:cafa274c:::benchmark_data_smithi064_185391_object71427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11613950899999, 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.189277785, 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71428 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148044+0000", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148045+0000", 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6899999999999998e-07 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148082+0000", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7800999999999999e-05 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148089+0000", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2349999999999996e-06 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148093+0000", 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4490000000000001e-06 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150147+0000", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020543169999999999 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150153+0000", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7699999999999998e-06 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318492+0000", 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1683386149999999 2024-08-17T17:34:47.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318638+0000", 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145742 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.318823+0000", 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018568299999999999 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321525+0000", 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027019539999999999 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.321558+0000", 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2984000000000001e-05 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.332861+0000", 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011302553 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336256+0000", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033948149999999998 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336311+0000", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4963000000000002e-05 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337322+0000", 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001011435 2024-08-17T17:34:47.321 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71439 6.13 6:cdd6e9ae:::benchmark_data_smithi064_185391_object71438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11608477499999, 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.190434312, 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71439 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148099+0000", 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5999999999999999e-07 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148138+0000", 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8883e-05 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148145+0000", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.28e-06 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148149+0000", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2899999999999996e-06 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150175+0000", 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020258120000000001 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150180+0000", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6080000000000003e-06 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.319903+0000", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.169723114 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.320064+0000", 2024-08-17T17:34:47.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000160941 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.320236+0000", 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017170900000000001 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.324936+0000", 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046998070000000003 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334730+0000", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097942949999999997 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.334747+0000", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6861e-05 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337424+0000", 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026770560000000001 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337484+0000", 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0291999999999998e-05 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338533+0000", 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001048904 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.329 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71448 6.13 6:c94d4d39:::benchmark_data_smithi064_185391_object71447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11602927400003, 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.194234171, 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71448 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148154+0000", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148155+0000", 2024-08-17T17:34:47.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2199999999999999e-07 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148193+0000", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7851000000000001e-05 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148199+0000", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3389999999999998e-06 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148203+0000", 2024-08-17T17:34:47.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2740000000000001e-06 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150202+0000", 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019990170000000001 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150208+0000", 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7919999999999998e-06 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326405+0000", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1761970420000001 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.326548+0000", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142993 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.329864+0000", 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033157310000000001 2024-08-17T17:34:47.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336134+0000", 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062696519999999997 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336181+0000", 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7429000000000003e-05 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336218+0000", 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7048999999999997e-05 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.340929+0000", 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047111210000000004 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.340985+0000", 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5849e-05 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342389+0000", 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00140361 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71459 6.13 6:ca93aa2f:::benchmark_data_smithi064_185391_object71458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11597388400003, 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.211085977, 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.337 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71459 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148210+0000", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6699999999999999e-07 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148249+0000", 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8473999999999998e-05 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148256+0000", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9110000000000004e-06 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148260+0000", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3370000000000001e-06 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150230+0000", 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019701839999999998 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150236+0000", 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8279999999999999e-06 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331258+0000", 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1810224380000001 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331409+0000", 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015012099999999999 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.331573+0000", 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164191 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.336140+0000", 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045669209999999998 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.337372+0000", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012318190000000001 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.343959+0000", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065878200000000003 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.357255+0000", 2024-08-17T17:34:47.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013295493 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.357804+0000", 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054940100000000003 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359296+0000", 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001491572 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.344 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71470 6.13 6:ce05dbf9:::benchmark_data_smithi064_185391_object71469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.115917486, 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.216359454, 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71470 2024-08-17T17:34:47.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148266+0000", 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148267+0000", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2e-07 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148305+0000", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8278999999999997e-05 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148311+0000", 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8769999999999998e-06 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148315+0000", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1010000000000002e-06 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150258+0000", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001942928 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150264+0000", 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7389999999999996e-06 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338570+0000", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.188306224 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338721+0000", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151605 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.338895+0000", 2024-08-17T17:34:47.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173902 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342442+0000", 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035463809999999999 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353252+0000", 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010810637 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356482+0000", 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032296260000000002 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359645+0000", 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031634190000000002 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359807+0000", 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000161846 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364626+0000", 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00481847 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71504 6.13 6:ca612f25:::benchmark_data_smithi064_185391_object71503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11586258900002, 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217744341, 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.352 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71504 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148321+0000", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7e-07 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148362+0000", 2024-08-17T17:34:47.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0194000000000002e-05 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148368+0000", 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8810000000000001e-06 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148373+0000", 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6500000000000001e-06 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150286+0000", 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001912871 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150292+0000", 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7119999999999997e-06 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342474+0000", 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1921820140000001 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342614+0000", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014014600000000001 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.342779+0000", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016551000000000001 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348286+0000", 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055062699999999997 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356468+0000", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081821329999999994 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356488+0000", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0157e-05 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364686+0000", 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081976570000000006 2024-08-17T17:34:47.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.364747+0000", 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1583999999999999e-05 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366065+0000", 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013181919999999999 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71518 6.13 6:cca05011:::benchmark_data_smithi064_185391_object71517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11580363799999, 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.22094334, 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71518 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148380+0000", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148381+0000", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9200000000000001e-07 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148421+0000", 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0896999999999998e-05 2024-08-17T17:34:47.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148429+0000", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148435+0000", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3730000000000001e-06 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150319+0000", 2024-08-17T17:34:47.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018839639999999999 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150325+0000", 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.343995+0000", 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1936702260000001 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.344156+0000", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016101599999999999 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.344345+0000", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000189328 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348329+0000", 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039839280000000003 2024-08-17T17:34:47.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356494+0000", 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081646300000000008 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359431+0000", 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029367830000000001 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366120+0000", 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066888980000000004 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.366179+0000", 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9175000000000002e-05 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369323+0000", 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031445179999999998 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71568 6.13 6:ca8b9dca:::benchmark_data_smithi064_185391_object71567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11574248800002, 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.222653634, 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71568 2024-08-17T17:34:47.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148441+0000", 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148442+0000", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0900000000000002e-07 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148480+0000", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8139999999999997e-05 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148487+0000", 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.737e-06 2024-08-17T17:34:47.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148491+0000", 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6460000000000001e-06 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150352+0000", 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018606600000000001 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150358+0000", 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4760000000000003e-06 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.345703+0000", 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1953445629999999 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.345855+0000", 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015168199999999999 2024-08-17T17:34:47.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.346446+0000", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000591768 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353296+0000", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006849656 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356500+0000", 2024-08-17T17:34:47.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032040150000000002 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359502+0000", 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00300198 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369428+0000", 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099256210000000008 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369597+0000", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016884800000000001 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371095+0000", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014983329999999999 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71616 6.13 6:cdda852e:::benchmark_data_smithi064_185391_object71615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:34:47.375 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11568644300002, 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225571365, 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71616 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:34:47.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148497+0000", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148498+0000", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3300000000000003e-07 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148583+0000", 2024-08-17T17:34:47.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4808000000000005e-05 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148589+0000", 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8669999999999996e-06 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148596+0000", 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.024e-06 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150384+0000", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017877570000000001 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150390+0000", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348360+0000", 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.197969335 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348504+0000", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144337 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.348685+0000", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018043000000000001 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353338+0000", 2024-08-17T17:34:47.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046530759999999999 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356506+0000", 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031686169999999999 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.369382+0000", 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012875985 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371198+0000", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018155790000000001 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371260+0000", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2675999999999994e-05 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374069+0000", 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028081479999999999 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.382 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71674 6.13 6:c8c36f7a:::benchmark_data_smithi064_185391_object71673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11558011800003, 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2267917589999999, 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71674 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148604+0000", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9399999999999995e-07 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148644+0000", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9901000000000003e-05 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148651+0000", 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0539999999999997e-06 2024-08-17T17:34:47.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148656+0000", 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.758e-06 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150415+0000", 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017591340000000001 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150421+0000", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1090000000000003e-06 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.351716+0000", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2012946879999999 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.351875+0000", 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159304 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.352054+0000", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017908999999999999 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353381+0000", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013267559999999999 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.356512+0000", 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003131572 2024-08-17T17:34:47.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.371152+0000", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014639353000000001 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374179+0000", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00302769 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374334+0000", 2024-08-17T17:34:47.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154938 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.375395+0000", 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010609180000000001 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71683 6.13 6:cb56e99d:::benchmark_data_smithi064_185391_object71682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e186)", 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 467.11552216000001, 2024-08-17T17:34:47.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2329368380000001, 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71683 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:34:47.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148662+0000", 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1800000000000001e-07 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148703+0000", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.074e-05 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148709+0000", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2280000000000003e-06 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.148714+0000", 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3469999999999998e-06 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150524+0000", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018093339999999999 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.150540+0000", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6679000000000001e-05 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.353442+0000", 2024-08-17T17:34:47.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2029013989999999 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.354929+0000", 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014875380000000001 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.355153+0000", 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022341099999999999 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359447+0000", 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042943510000000001 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.359556+0000", 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109091 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.374127+0000", 2024-08-17T17:34:47.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014571183999999999 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.379151+0000", 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005024204 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.379515+0000", 2024-08-17T17:34:47.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000363734 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:01.381598+0000", 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00208318 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:134208 7.e 7:70fc098c:::benchmark_data_smithi064_210574_object35399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:34:47.399 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 135.927416916, 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33046408199999999, 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134208 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336768+0000", 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3099999999999997e-07 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336772+0000", 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0239999999999999e-06 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336780+0000", 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9060000000000008e-06 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336787+0000", 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9979999999999997e-06 2024-08-17T17:34:47.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336906+0000", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118996 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.337017+0000", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011017 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.337122+0000", 2024-08-17T17:34:47.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104943 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.341658+0000", 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045360569999999996 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.342165+0000", 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050721900000000005 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667129+0000", 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.324964268 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667190+0000", 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0482000000000001e-05 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667231+0000", 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1288000000000002e-05 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:171371 7.e 7:7e7bcdb4:::benchmark_data_smithi064_210574_object72562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 79.416349507999996, 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18653573200000001, 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.406 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171371 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847836+0000", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847842+0000", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9869999999999996e-06 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847855+0000", 2024-08-17T17:34:47.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3108999999999999e-05 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847862+0000", 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.362e-06 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848527+0000", 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066430000000000005 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848665+0000", 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013854899999999999 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848792+0000", 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012693200000000001 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860535+0000", 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011742452 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860549+0000", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4796e-05 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034262+0000", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17371302199999999 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034325+0000", 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2119000000000006e-05 2024-08-17T17:34:47.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034370+0000", 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5374999999999998e-05 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:171373 7.e 7:79305286:::benchmark_data_smithi064_210574_object72564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 79.411559667000006, 2024-08-17T17:34:47.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.191019191, 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171373 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852626+0000", 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8309999999999999e-06 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852632+0000", 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6790000000000002e-06 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852645+0000", 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3906e-05 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852658+0000", 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2113000000000001e-05 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854505+0000", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00184786 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854693+0000", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018749899999999999 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854819+0000", 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012632299999999999 2024-08-17T17:34:47.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860872+0000", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060525919999999999 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860954+0000", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1791000000000003e-05 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043544+0000", 2024-08-17T17:34:47.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18258986299999999 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043601+0000", 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7256999999999997e-05 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043643+0000", 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2477e-05 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.418 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193378 7.e 7:736e894c:::benchmark_data_smithi064_210574_object94569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.210661057999999, 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18333747, 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.419 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193378 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8599999999999998e-07 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053525+0000", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2649999999999999e-06 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053532+0000", 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4060000000000002e-06 2024-08-17T17:34:47.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053537+0000", 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8389999999999997e-06 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.062906+0000", 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093698259999999995 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.063009+0000", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102575 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.063126+0000", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011680699999999999 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073581+0000", 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010454857 2024-08-17T17:34:47.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083008+0000", 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094271070000000005 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236755+0000", 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15374724200000001 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236812+0000", 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7408999999999999e-05 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236860+0000", 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7651000000000002e-05 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.425 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193384 7.e 7:79d10fe6:::benchmark_data_smithi064_210574_object94575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.210171801999998, 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18298474300000001, 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193384 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.426 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054013+0000", 2024-08-17T17:34:47.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054017+0000", 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.216e-06 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054028+0000", 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0789e-05 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054034+0000", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3589999999999996e-06 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068303+0000", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014269380999999999 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068428+0000", 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124456 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068547+0000", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119392 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083020+0000", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014472766999999999 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083050+0000", 2024-08-17T17:34:47.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0168e-05 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236913+0000", 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.153863004 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236957+0000", 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3389000000000001e-05 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236997+0000", 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0036000000000001e-05 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193392 7.e 7:7ca26a87:::benchmark_data_smithi064_210574_object94583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.195704939000002, 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16874679300000001, 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.432 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193392 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068480+0000", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.251e-06 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068487+0000", 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:34:47.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068499+0000", 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2802e-05 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068507+0000", 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1210000000000007e-06 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073606+0000", 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050989970000000001 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073707+0000", 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010047000000000001 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073808+0000", 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010090900000000001 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083091+0000", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092827120000000003 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083102+0000", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.095e-05 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237137+0000", 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15403577900000001 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237186+0000", 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9146e-05 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237226+0000", 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9125999999999998e-05 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35217.0:193405 7.a e269/242 7:5373d8f6:::benchmark_data_smithi064_210574_object94596:head v 269'12007, mlcod=269'12005)", 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.175563584000002, 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16314719699999999, 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088622+0000", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7e-06 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088630+0000", 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5669999999999995e-06 2024-08-17T17:34:47.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236740+0000", 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.148109451 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236753+0000", 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.292e-05 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:34:47.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237161+0000", 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040819599999999999 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237183+0000", 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2277999999999999e-05 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.251722+0000", 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014538656 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:34:47.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.251767+0000", 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5429000000000002e-05 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:34:47.443 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:34:47.450 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' created 2024-08-17T17:34:47.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:47.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:47.594 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:47 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1910771350' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:47 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1910771350' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:47.717 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:47.819 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:47.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:47 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1910771350' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:34:47.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:47.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.964 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:47.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:48.087 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:34:48.122 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1271,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":274,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915895,"num_in_osds":8,"osd_in_since":1723915756,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":28,"data_bytes":4790245,"bytes_used":3341692928,"bytes_avail":764618592256,"bytes_total":767960285184},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-17T17:34:21.105498+0000","services":{"osd":{"daemons":{"summary":"","3":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-08-17T17:34:48.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:48.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:48.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:48.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:48.335 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:48.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:48.568 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:48.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:48.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:48.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:48.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.861 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:48 smithi175 ceph-mon[101434]: pgmap v887: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:48.861 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:48 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:34:48.861 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:48 smithi175 ceph-mon[101434]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:34:48.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:48 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:48.862 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1196412368' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:34:48.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[104516]: pgmap v887: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:48.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:34:48.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[104516]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:34:48.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1196412368' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[109246]: pgmap v887: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[109246]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:48.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1196412368' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:34:48.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:48.958 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:48.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:49.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:49.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:49.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:49.208 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:49.310 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-08-17T17:34:49.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:34:49.310 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:49.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:49.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:49.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:49.561 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:49.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:49.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:49.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:49 smithi064 ceph-mon[104516]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:34:49.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:49 smithi064 ceph-mon[109246]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:34:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:49.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:49.740 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:49 smithi175 ceph-mon[101434]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:34:49.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:34:49.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:49.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:49.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:49.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:50.079 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:50.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:50.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.327 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:50.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:50.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:50.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:50.451 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:50.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:50.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:50.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:50.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[104516]: pgmap v890: 89 pgs: 13 unknown, 76 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[104516]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:34:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[109246]: pgmap v890: 89 pgs: 13 unknown, 76 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:50 smithi064 ceph-mon[109246]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:34:50.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:50.711 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:50 smithi175 ceph-mon[101434]: pgmap v890: 89 pgs: 13 unknown, 76 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:50.711 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:50 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:50.711 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:50 smithi175 ceph-mon[101434]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:34:50.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:50.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-08-17T17:34:50.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:50.955 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.057 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:51.072 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:51.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:51.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:51.181 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.204 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:51.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.305 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:51.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:51.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:51.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:51.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:51.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:51.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:51.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:51.822 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:51.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:51.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:52.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:52.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:52.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:52.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:52.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:52.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:52.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.609 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:52.616 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:34:52.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:52.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:52.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:52.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:52 smithi175 ceph-mon[101434]: pgmap v892: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:52.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[104516]: pgmap v892: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[109246]: pgmap v892: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:52.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:53.067 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:53.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:53.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:53.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:53.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:53.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:53.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:53.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:53.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:53.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:53.741 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:53.741 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:53.742 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[104516]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:34:53.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:53.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:53.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-mon[109246]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:34:53.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:53.813 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:53 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/72374042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:53 smithi175 ceph-mon[101434]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:34:53.915 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:53.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:54.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:54.046 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:34:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:34:53] "GET /metrics HTTP/1.1" 200 34287 "" "Prometheus/2.51.0" 2024-08-17T17:34:54.067 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:54.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:54.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:54.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:54.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:54.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:54.548 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:54.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:54.634 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:54.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:54 smithi064 ceph-mon[104516]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:54 smithi064 ceph-mon[109246]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:54.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:54.801 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:34:54.802 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:34:54.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:54.827 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:54 smithi175 ceph-mon[101434]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:54.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:54.950 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:55.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:55.078 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:55.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:55.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:55.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:55.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:55.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:55.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.450 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:55.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:55.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:55.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:55.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:55.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:34:55.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:55.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:55.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:56.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:56.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:56.190 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:56.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:56.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:56 smithi175 ceph-mon[101434]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:56 smithi175 ceph-mon[101434]: pgmap v895: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[104516]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[104516]: pgmap v895: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[109246]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:56 smithi064 ceph-mon[109246]: pgmap v895: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:56.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:56.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:56.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:56.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:56.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:56.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:56.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:56.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:56.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:56.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:56.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:57.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.364 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":894,"stamp":"2024-08-17T17:34:53.112691+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":10166083,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":245,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3263628,"kb_used_data":41496,"kb_used_omap":46,"kb_used_meta":3221969,"kb_avail":746697588,"statfs":{"total":767960285184,"available":764618330112,"internally_reserved":0,"allocated":42491904,"data_stored":30693293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":3299296499},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.669684"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":407,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.452639+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:34:48.452639+0000","last_peered":"2024-08-17T17:34:48.452639+0000","last_clean":"2024-08-17T17:34:48.452639+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:34:48.452639+0000","last_undegraded":"2024-08-17T17:34:48.452639+0000","last_fullsized":"2024-08-17T17:34:48.452639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":273,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.460690+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:34:48.460690+0000","last_peered":"2024-08-17T17:34:48.460690+0000","last_clean":"2024-08-17T17:34:48.460690+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:34:48.460690+0000","last_undegraded":"2024-08-17T17:34:48.460690+0000","last_fullsized":"2024-08-17T17:34:48.460690+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":57,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446190+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:34:48.446190+0000","last_peered":"2024-08-17T17:34:48.446190+0000","last_clean":"2024-08-17T17:34:48.446190+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:34:48.446190+0000","last_undegraded":"2024-08-17T17:34:48.446190+0000","last_fullsized":"2024-08-17T17:34:48.446190+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":399,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450099+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:34:48.450099+0000","last_peered":"2024-08-17T17:34:48.450099+0000","last_clean":"2024-08-17T17:34:48.450099+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:34:48.450099+0000","last_undegraded":"2024-08-17T17:34:48.450099+0000","last_fullsized":"2024-08-17T17:34:48.450099+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450801+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:34:48.450801+0000","last_peered":"2024-08-17T17:34:48.450801+0000","last_clean":"2024-08-17T17:34:48.450801+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:34:48.450801+0000","last_undegraded":"2024-08-17T17:34:48.450801+0000","last_fullsized":"2024-08-17T17:34:48.450801+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":259,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.460484+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:34:48.460484+0000","last_peered":"2024-08-17T17:34:48.460484+0000","last_clean":"2024-08-17T17:34:48.460484+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:34:48.460484+0000","last_undegraded":"2024-08-17T17:34:48.460484+0000","last_fullsized":"2024-08-17T17:34:48.460484+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":407,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.453908+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:34:48.453908+0000","last_peered":"2024-08-17T17:34:48.453908+0000","last_clean":"2024-08-17T17:34:48.453908+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:34:48.453908+0000","last_undegraded":"2024-08-17T17:34:48.453908+0000","last_fullsized":"2024-08-17T17:34:48.453908+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":206,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-17T17:34:47.446318+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:34:47.446318+0000","last_peered":"2024-08-17T17:34:47.446318+0000","last_clean":"2024-08-17T17:34:47.446318+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:34:47.446318+0000","last_undegraded":"2024-08-17T17:34:47.446318+0000","last_fullsized":"2024-08-17T17:34:47.446318+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":11,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461741+0000","last_change":"2024-08-17T17:34:48.462975+0000","last_active":"2024-08-17T17:34:49.461741+0000","last_peered":"2024-08-17T17:34:49.461741+0000","last_clean":"2024-08-17T17:34:49.461741+0000","last_became_active":"2024-08-17T17:34:48.462675+0000","last_became_peered":"2024-08-17T17:34:48.462675+0000","last_unstale":"2024-08-17T17:34:49.461741+0000","last_undegraded":"2024-08-17T17:34:49.461741+0000","last_fullsized":"2024-08-17T17:34:49.461741+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:53.644325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":268,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.461132+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:34:48.461132+0000","last_peered":"2024-08-17T17:34:48.461132+0000","last_clean":"2024-08-17T17:34:48.461132+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:34:48.461132+0000","last_undegraded":"2024-08-17T17:34:48.461132+0000","last_fullsized":"2024-08-17T17:34:48.461132+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":67,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446361+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:34:48.446361+0000","last_peered":"2024-08-17T17:34:48.446361+0000","last_clean":"2024-08-17T17:34:48.446361+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:34:48.446361+0000","last_undegraded":"2024-08-17T17:34:48.446361+0000","last_fullsized":"2024-08-17T17:34:48.446361+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":10,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.461345+0000","last_change":"2024-08-17T17:34:48.460978+0000","last_active":"2024-08-17T17:34:48.461345+0000","last_peered":"2024-08-17T17:34:48.461345+0000","last_clean":"2024-08-17T17:34:48.461345+0000","last_became_active":"2024-08-17T17:34:48.460684+0000","last_became_peered":"2024-08-17T17:34:48.460684+0000","last_unstale":"2024-08-17T17:34:48.461345+0000","last_undegraded":"2024-08-17T17:34:48.461345+0000","last_fullsized":"2024-08-17T17:34:48.461345+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:51.491253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":270,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461707+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:34:49.461707+0000","last_peered":"2024-08-17T17:34:49.461707+0000","last_clean":"2024-08-17T17:34:49.461707+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:34:49.461707+0000","last_undegraded":"2024-08-17T17:34:49.461707+0000","last_fullsized":"2024-08-17T17:34:49.461707+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450562+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:34:48.450562+0000","last_peered":"2024-08-17T17:34:48.450562+0000","last_clean":"2024-08-17T17:34:48.450562+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:34:48.450562+0000","last_undegraded":"2024-08-17T17:34:48.450562+0000","last_fullsized":"2024-08-17T17:34:48.450562+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.461704+0000","last_change":"2024-08-17T17:34:48.460837+0000","last_active":"2024-08-17T17:34:48.461704+0000","last_peered":"2024-08-17T17:34:48.461704+0000","last_clean":"2024-08-17T17:34:48.461704+0000","last_became_active":"2024-08-17T17:34:48.460556+0000","last_became_peered":"2024-08-17T17:34:48.460556+0000","last_unstale":"2024-08-17T17:34:48.461704+0000","last_undegraded":"2024-08-17T17:34:48.461704+0000","last_fullsized":"2024-08-17T17:34:48.461704+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:37.247364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":287,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461734+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:34:49.461734+0000","last_peered":"2024-08-17T17:34:49.461734+0000","last_clean":"2024-08-17T17:34:49.461734+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:34:49.461734+0000","last_undegraded":"2024-08-17T17:34:49.461734+0000","last_fullsized":"2024-08-17T17:34:49.461734+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":186,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-17T17:34:47.446725+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:34:47.446725+0000","last_peered":"2024-08-17T17:34:47.446725+0000","last_clean":"2024-08-17T17:34:47.446725+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:34:47.446725+0000","last_undegraded":"2024-08-17T17:34:47.446725+0000","last_fullsized":"2024-08-17T17:34:47.446725+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":14,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579847+0000","last_change":"2024-08-17T17:34:48.460872+0000","last_active":"2024-08-17T17:34:49.579847+0000","last_peered":"2024-08-17T17:34:49.579847+0000","last_clean":"2024-08-17T17:34:49.579847+0000","last_became_active":"2024-08-17T17:34:48.460576+0000","last_became_peered":"2024-08-17T17:34:48.460576+0000","last_unstale":"2024-08-17T17:34:49.579847+0000","last_undegraded":"2024-08-17T17:34:49.579847+0000","last_fullsized":"2024-08-17T17:34:49.579847+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:44.349399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":67,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446135+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:34:48.446135+0000","last_peered":"2024-08-17T17:34:48.446135+0000","last_clean":"2024-08-17T17:34:48.446135+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:34:48.446135+0000","last_undegraded":"2024-08-17T17:34:48.446135+0000","last_fullsized":"2024-08-17T17:34:48.446135+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":413,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450823+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:34:48.450823+0000","last_peered":"2024-08-17T17:34:48.450823+0000","last_clean":"2024-08-17T17:34:48.450823+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:34:48.450823+0000","last_undegraded":"2024-08-17T17:34:48.450823+0000","last_fullsized":"2024-08-17T17:34:48.450823+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.459736+0000","last_change":"2024-08-17T17:34:48.458793+0000","last_active":"2024-08-17T17:34:48.459736+0000","last_peered":"2024-08-17T17:34:48.459736+0000","last_clean":"2024-08-17T17:34:48.459736+0000","last_became_active":"2024-08-17T17:34:48.458438+0000","last_became_peered":"2024-08-17T17:34:48.458438+0000","last_unstale":"2024-08-17T17:34:48.459736+0000","last_undegraded":"2024-08-17T17:34:48.459736+0000","last_fullsized":"2024-08-17T17:34:48.459736+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:12:45.085734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":429,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.452376+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:34:48.452376+0000","last_peered":"2024-08-17T17:34:48.452376+0000","last_clean":"2024-08-17T17:34:48.452376+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:34:48.452376+0000","last_undegraded":"2024-08-17T17:34:48.452376+0000","last_fullsized":"2024-08-17T17:34:48.452376+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450931+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:34:48.450931+0000","last_peered":"2024-08-17T17:34:48.450931+0000","last_clean":"2024-08-17T17:34:48.450931+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:34:48.450931+0000","last_undegraded":"2024-08-17T17:34:48.450931+0000","last_fullsized":"2024-08-17T17:34:48.450931+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":14,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579106+0000","last_change":"2024-08-17T17:34:48.463240+0000","last_active":"2024-08-17T17:34:49.579106+0000","last_peered":"2024-08-17T17:34:49.579106+0000","last_clean":"2024-08-17T17:34:49.579106+0000","last_became_active":"2024-08-17T17:34:48.462973+0000","last_became_peered":"2024-08-17T17:34:48.462973+0000","last_unstale":"2024-08-17T17:34:49.579106+0000","last_undegraded":"2024-08-17T17:34:49.579106+0000","last_fullsized":"2024-08-17T17:34:49.579106+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:30:49.967045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":194,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450503+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:34:48.450503+0000","last_peered":"2024-08-17T17:34:48.450503+0000","last_clean":"2024-08-17T17:34:48.450503+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:34:48.450503+0000","last_undegraded":"2024-08-17T17:34:48.450503+0000","last_fullsized":"2024-08-17T17:34:48.450503+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":164,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451029+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:34:48.451029+0000","last_peered":"2024-08-17T17:34:48.451029+0000","last_clean":"2024-08-17T17:34:48.451029+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:34:48.451029+0000","last_undegraded":"2024-08-17T17:34:48.451029+0000","last_fullsized":"2024-08-17T17:34:48.451029+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":11,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577222+0000","last_change":"2024-08-17T17:34:48.453342+0000","last_active":"2024-08-17T17:34:49.577222+0000","last_peered":"2024-08-17T17:34:49.577222+0000","last_clean":"2024-08-17T17:34:49.577222+0000","last_became_active":"2024-08-17T17:34:48.453113+0000","last_became_peered":"2024-08-17T17:34:48.453113+0000","last_unstale":"2024-08-17T17:34:49.577222+0000","last_undegraded":"2024-08-17T17:34:49.577222+0000","last_fullsized":"2024-08-17T17:34:49.577222+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:39.280205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450432+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:34:48.450432+0000","last_peered":"2024-08-17T17:34:48.450432+0000","last_clean":"2024-08-17T17:34:48.450432+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:34:48.450432+0000","last_undegraded":"2024-08-17T17:34:48.450432+0000","last_fullsized":"2024-08-17T17:34:48.450432+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450917+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:34:48.450917+0000","last_peered":"2024-08-17T17:34:48.450917+0000","last_clean":"2024-08-17T17:34:48.450917+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:34:48.450917+0000","last_undegraded":"2024-08-17T17:34:48.450917+0000","last_fullsized":"2024-08-17T17:34:48.450917+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":14,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579533+0000","last_change":"2024-08-17T17:34:48.461336+0000","last_active":"2024-08-17T17:34:49.579533+0000","last_peered":"2024-08-17T17:34:49.579533+0000","last_clean":"2024-08-17T17:34:49.579533+0000","last_became_active":"2024-08-17T17:34:48.461060+0000","last_became_peered":"2024-08-17T17:34:48.461060+0000","last_unstale":"2024-08-17T17:34:49.579533+0000","last_undegraded":"2024-08-17T17:34:49.579533+0000","last_fullsized":"2024-08-17T17:34:49.579533+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:41.329353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":409,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451390+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:34:48.451390+0000","last_peered":"2024-08-17T17:34:48.451390+0000","last_clean":"2024-08-17T17:34:48.451390+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:34:48.451390+0000","last_undegraded":"2024-08-17T17:34:48.451390+0000","last_fullsized":"2024-08-17T17:34:48.451390+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":399,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.578698+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:34:49.578698+0000","last_peered":"2024-08-17T17:34:49.578698+0000","last_clean":"2024-08-17T17:34:49.578698+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:34:49.578698+0000","last_undegraded":"2024-08-17T17:34:49.578698+0000","last_fullsized":"2024-08-17T17:34:49.578698+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":11,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461556+0000","last_change":"2024-08-17T17:34:48.455278+0000","last_active":"2024-08-17T17:34:49.461556+0000","last_peered":"2024-08-17T17:34:49.461556+0000","last_clean":"2024-08-17T17:34:49.461556+0000","last_became_active":"2024-08-17T17:34:48.455018+0000","last_became_peered":"2024-08-17T17:34:48.455018+0000","last_unstale":"2024-08-17T17:34:49.461556+0000","last_undegraded":"2024-08-17T17:34:49.461556+0000","last_fullsized":"2024-08-17T17:34:49.461556+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:53.287262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":589,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.460509+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:34:48.460509+0000","last_peered":"2024-08-17T17:34:48.460509+0000","last_clean":"2024-08-17T17:34:48.460509+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:34:48.460509+0000","last_undegraded":"2024-08-17T17:34:48.460509+0000","last_fullsized":"2024-08-17T17:34:48.460509+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":865,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446264+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:34:48.446264+0000","last_peered":"2024-08-17T17:34:48.446264+0000","last_clean":"2024-08-17T17:34:48.446264+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:34:48.446264+0000","last_undegraded":"2024-08-17T17:34:48.446264+0000","last_fullsized":"2024-08-17T17:34:48.446264+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":392,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451002+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:34:48.451002+0000","last_peered":"2024-08-17T17:34:48.451002+0000","last_clean":"2024-08-17T17:34:48.451002+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:34:48.451002+0000","last_undegraded":"2024-08-17T17:34:48.451002+0000","last_fullsized":"2024-08-17T17:34:48.451002+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":178,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450778+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:34:48.450778+0000","last_peered":"2024-08-17T17:34:48.450778+0000","last_clean":"2024-08-17T17:34:48.450778+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:34:48.450778+0000","last_undegraded":"2024-08-17T17:34:48.450778+0000","last_fullsized":"2024-08-17T17:34:48.450778+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":14,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577544+0000","last_change":"2024-08-17T17:34:48.462773+0000","last_active":"2024-08-17T17:34:49.577544+0000","last_peered":"2024-08-17T17:34:49.577544+0000","last_clean":"2024-08-17T17:34:49.577544+0000","last_became_active":"2024-08-17T17:34:48.462475+0000","last_became_peered":"2024-08-17T17:34:48.462475+0000","last_unstale":"2024-08-17T17:34:49.577544+0000","last_undegraded":"2024-08-17T17:34:49.577544+0000","last_fullsized":"2024-08-17T17:34:49.577544+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:07.248087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":652,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.460505+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:34:48.460505+0000","last_peered":"2024-08-17T17:34:48.460505+0000","last_clean":"2024-08-17T17:34:48.460505+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:34:48.460505+0000","last_undegraded":"2024-08-17T17:34:48.460505+0000","last_fullsized":"2024-08-17T17:34:48.460505+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":414,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451779+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:34:48.451779+0000","last_peered":"2024-08-17T17:34:48.451779+0000","last_clean":"2024-08-17T17:34:48.451779+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:34:48.451779+0000","last_undegraded":"2024-08-17T17:34:48.451779+0000","last_fullsized":"2024-08-17T17:34:48.451779+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":411,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450981+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:34:48.450981+0000","last_peered":"2024-08-17T17:34:48.450981+0000","last_clean":"2024-08-17T17:34:48.450981+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:34:48.450981+0000","last_undegraded":"2024-08-17T17:34:48.450981+0000","last_fullsized":"2024-08-17T17:34:48.450981+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.459427+0000","last_change":"2024-08-17T17:34:48.457912+0000","last_active":"2024-08-17T17:34:48.459427+0000","last_peered":"2024-08-17T17:34:48.459427+0000","last_clean":"2024-08-17T17:34:48.459427+0000","last_became_active":"2024-08-17T17:34:48.457628+0000","last_became_peered":"2024-08-17T17:34:48.457628+0000","last_unstale":"2024-08-17T17:34:48.459427+0000","last_undegraded":"2024-08-17T17:34:48.459427+0000","last_fullsized":"2024-08-17T17:34:48.459427+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:42.534100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":813,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577774+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:34:49.577774+0000","last_peered":"2024-08-17T17:34:49.577774+0000","last_clean":"2024-08-17T17:34:49.577774+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:34:49.577774+0000","last_undegraded":"2024-08-17T17:34:49.577774+0000","last_fullsized":"2024-08-17T17:34:49.577774+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":405,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579185+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:34:49.579185+0000","last_peered":"2024-08-17T17:34:49.579185+0000","last_clean":"2024-08-17T17:34:49.579185+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:34:49.579185+0000","last_undegraded":"2024-08-17T17:34:49.579185+0000","last_fullsized":"2024-08-17T17:34:49.579185+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":452,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579770+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:34:49.579770+0000","last_peered":"2024-08-17T17:34:49.579770+0000","last_clean":"2024-08-17T17:34:49.579770+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:34:49.579770+0000","last_undegraded":"2024-08-17T17:34:49.579770+0000","last_fullsized":"2024-08-17T17:34:49.579770+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":11,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461518+0000","last_change":"2024-08-17T17:34:48.462500+0000","last_active":"2024-08-17T17:34:49.461518+0000","last_peered":"2024-08-17T17:34:49.461518+0000","last_clean":"2024-08-17T17:34:49.461518+0000","last_became_active":"2024-08-17T17:34:48.462215+0000","last_became_peered":"2024-08-17T17:34:48.462215+0000","last_unstale":"2024-08-17T17:34:49.461518+0000","last_undegraded":"2024-08-17T17:34:49.461518+0000","last_fullsized":"2024-08-17T17:34:49.461518+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:31:21.306309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":988,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.445664+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:34:48.445664+0000","last_peered":"2024-08-17T17:34:48.445664+0000","last_clean":"2024-08-17T17:34:48.445664+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:34:48.445664+0000","last_undegraded":"2024-08-17T17:34:48.445664+0000","last_fullsized":"2024-08-17T17:34:48.445664+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":231,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577879+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:34:49.577879+0000","last_peered":"2024-08-17T17:34:49.577879+0000","last_clean":"2024-08-17T17:34:49.577879+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:34:49.577879+0000","last_undegraded":"2024-08-17T17:34:49.577879+0000","last_fullsized":"2024-08-17T17:34:49.577879+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":299,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579473+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:34:49.579473+0000","last_peered":"2024-08-17T17:34:49.579473+0000","last_clean":"2024-08-17T17:34:49.579473+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:34:49.579473+0000","last_undegraded":"2024-08-17T17:34:49.579473+0000","last_fullsized":"2024-08-17T17:34:49.579473+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.465148+0000","last_change":"2024-08-17T17:34:48.464724+0000","last_active":"2024-08-17T17:34:48.465148+0000","last_peered":"2024-08-17T17:34:48.465148+0000","last_clean":"2024-08-17T17:34:48.465148+0000","last_became_active":"2024-08-17T17:34:48.464408+0000","last_became_peered":"2024-08-17T17:34:48.464408+0000","last_unstale":"2024-08-17T17:34:48.465148+0000","last_undegraded":"2024-08-17T17:34:48.465148+0000","last_fullsized":"2024-08-17T17:34:48.465148+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:14.479184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":57,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446110+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:34:48.446110+0000","last_peered":"2024-08-17T17:34:48.446110+0000","last_clean":"2024-08-17T17:34:48.446110+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:34:48.446110+0000","last_undegraded":"2024-08-17T17:34:48.446110+0000","last_fullsized":"2024-08-17T17:34:48.446110+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":698,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579578+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:34:49.579578+0000","last_peered":"2024-08-17T17:34:49.579578+0000","last_clean":"2024-08-17T17:34:49.579578+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:34:49.579578+0000","last_undegraded":"2024-08-17T17:34:49.579578+0000","last_fullsized":"2024-08-17T17:34:49.579578+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":422,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579413+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:34:49.579413+0000","last_peered":"2024-08-17T17:34:49.579413+0000","last_clean":"2024-08-17T17:34:49.579413+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:34:49.579413+0000","last_undegraded":"2024-08-17T17:34:49.579413+0000","last_fullsized":"2024-08-17T17:34:49.579413+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.464469+0000","last_change":"2024-08-17T17:34:48.463247+0000","last_active":"2024-08-17T17:34:48.464469+0000","last_peered":"2024-08-17T17:34:48.464469+0000","last_clean":"2024-08-17T17:34:48.464469+0000","last_became_active":"2024-08-17T17:34:48.462966+0000","last_became_peered":"2024-08-17T17:34:48.462966+0000","last_unstale":"2024-08-17T17:34:48.464469+0000","last_undegraded":"2024-08-17T17:34:48.464469+0000","last_fullsized":"2024-08-17T17:34:48.464469+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:17:32.786869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":419,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579539+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:34:49.579539+0000","last_peered":"2024-08-17T17:34:49.579539+0000","last_clean":"2024-08-17T17:34:49.579539+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:34:49.579539+0000","last_undegraded":"2024-08-17T17:34:49.579539+0000","last_fullsized":"2024-08-17T17:34:49.579539+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":841,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.461277+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:34:48.461277+0000","last_peered":"2024-08-17T17:34:48.461277+0000","last_clean":"2024-08-17T17:34:48.461277+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:34:48.461277+0000","last_undegraded":"2024-08-17T17:34:48.461277+0000","last_fullsized":"2024-08-17T17:34:48.461277+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":414,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450737+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:34:48.450737+0000","last_peered":"2024-08-17T17:34:48.450737+0000","last_clean":"2024-08-17T17:34:48.450737+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:34:48.450737+0000","last_undegraded":"2024-08-17T17:34:48.450737+0000","last_fullsized":"2024-08-17T17:34:48.450737+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":11,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.464169+0000","last_change":"2024-08-17T17:34:48.462499+0000","last_active":"2024-08-17T17:34:48.464169+0000","last_peered":"2024-08-17T17:34:48.464169+0000","last_clean":"2024-08-17T17:34:48.464169+0000","last_became_active":"2024-08-17T17:34:48.461996+0000","last_became_peered":"2024-08-17T17:34:48.461996+0000","last_unstale":"2024-08-17T17:34:48.464169+0000","last_undegraded":"2024-08-17T17:34:48.464169+0000","last_fullsized":"2024-08-17T17:34:48.464169+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:54.794535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":407,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.453801+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:34:48.453801+0000","last_peered":"2024-08-17T17:34:48.453801+0000","last_clean":"2024-08-17T17:34:48.453801+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:34:48.453801+0000","last_undegraded":"2024-08-17T17:34:48.453801+0000","last_fullsized":"2024-08-17T17:34:48.453801+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":928,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579751+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:34:49.579751+0000","last_peered":"2024-08-17T17:34:49.579751+0000","last_clean":"2024-08-17T17:34:49.579751+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:34:49.579751+0000","last_undegraded":"2024-08-17T17:34:49.579751+0000","last_fullsized":"2024-08-17T17:34:49.579751+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":404,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451216+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:34:48.451216+0000","last_peered":"2024-08-17T17:34:48.451216+0000","last_clean":"2024-08-17T17:34:48.451216+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:34:48.451216+0000","last_undegraded":"2024-08-17T17:34:48.451216+0000","last_fullsized":"2024-08-17T17:34:48.451216+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":11,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.461852+0000","last_change":"2024-08-17T17:34:48.463451+0000","last_active":"2024-08-17T17:34:49.461852+0000","last_peered":"2024-08-17T17:34:49.461852+0000","last_clean":"2024-08-17T17:34:49.461852+0000","last_became_active":"2024-08-17T17:34:48.463124+0000","last_became_peered":"2024-08-17T17:34:48.463124+0000","last_unstale":"2024-08-17T17:34:49.461852+0000","last_undegraded":"2024-08-17T17:34:49.461852+0000","last_fullsized":"2024-08-17T17:34:49.461852+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:58:26.982388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":426,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450697+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:34:48.450697+0000","last_peered":"2024-08-17T17:34:48.450697+0000","last_clean":"2024-08-17T17:34:48.450697+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:34:48.450697+0000","last_undegraded":"2024-08-17T17:34:48.450697+0000","last_fullsized":"2024-08-17T17:34:48.450697+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":397,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.461207+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:34:48.461207+0000","last_peered":"2024-08-17T17:34:48.461207+0000","last_clean":"2024-08-17T17:34:48.461207+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:34:48.461207+0000","last_undegraded":"2024-08-17T17:34:48.461207+0000","last_fullsized":"2024-08-17T17:34:48.461207+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577463+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:34:49.577463+0000","last_peered":"2024-08-17T17:34:49.577463+0000","last_clean":"2024-08-17T17:34:49.577463+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:34:49.577463+0000","last_undegraded":"2024-08-17T17:34:49.577463+0000","last_fullsized":"2024-08-17T17:34:49.577463+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451112+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:34:48.451112+0000","last_peered":"2024-08-17T17:34:48.451112+0000","last_clean":"2024-08-17T17:34:48.451112+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:34:48.451112+0000","last_undegraded":"2024-08-17T17:34:48.451112+0000","last_fullsized":"2024-08-17T17:34:48.451112+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":189,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450852+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:34:48.450852+0000","last_peered":"2024-08-17T17:34:48.450852+0000","last_clean":"2024-08-17T17:34:48.450852+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:34:48.450852+0000","last_undegraded":"2024-08-17T17:34:48.450852+0000","last_fullsized":"2024-08-17T17:34:48.450852+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":404,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579335+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:34:49.579335+0000","last_peered":"2024-08-17T17:34:49.579335+0000","last_clean":"2024-08-17T17:34:49.579335+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:34:49.579335+0000","last_undegraded":"2024-08-17T17:34:49.579335+0000","last_fullsized":"2024-08-17T17:34:49.579335+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":274,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.460926+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:34:48.460926+0000","last_peered":"2024-08-17T17:34:48.460926+0000","last_clean":"2024-08-17T17:34:48.460926+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:34:48.460926+0000","last_undegraded":"2024-08-17T17:34:48.460926+0000","last_fullsized":"2024-08-17T17:34:48.460926+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":57,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446080+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:34:48.446080+0000","last_peered":"2024-08-17T17:34:48.446080+0000","last_clean":"2024-08-17T17:34:48.446080+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:34:48.446080+0000","last_undegraded":"2024-08-17T17:34:48.446080+0000","last_fullsized":"2024-08-17T17:34:48.446080+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":369,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451236+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:34:48.451236+0000","last_peered":"2024-08-17T17:34:48.451236+0000","last_clean":"2024-08-17T17:34:48.451236+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:34:48.451236+0000","last_undegraded":"2024-08-17T17:34:48.451236+0000","last_fullsized":"2024-08-17T17:34:48.451236+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":428,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451679+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:34:48.451679+0000","last_peered":"2024-08-17T17:34:48.451679+0000","last_clean":"2024-08-17T17:34:48.451679+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:34:48.451679+0000","last_undegraded":"2024-08-17T17:34:48.451679+0000","last_fullsized":"2024-08-17T17:34:48.451679+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":418,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451132+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:34:48.451132+0000","last_peered":"2024-08-17T17:34:48.451132+0000","last_clean":"2024-08-17T17:34:48.451132+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:34:48.451132+0000","last_undegraded":"2024-08-17T17:34:48.451132+0000","last_fullsized":"2024-08-17T17:34:48.451132+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446013+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:34:48.446013+0000","last_peered":"2024-08-17T17:34:48.446013+0000","last_clean":"2024-08-17T17:34:48.446013+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:34:48.446013+0000","last_undegraded":"2024-08-17T17:34:48.446013+0000","last_fullsized":"2024-08-17T17:34:48.446013+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":423,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451300+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:34:48.451300+0000","last_peered":"2024-08-17T17:34:48.451300+0000","last_clean":"2024-08-17T17:34:48.451300+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:34:48.451300+0000","last_undegraded":"2024-08-17T17:34:48.451300+0000","last_fullsized":"2024-08-17T17:34:48.451300+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":400,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450903+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:34:48.450903+0000","last_peered":"2024-08-17T17:34:48.450903+0000","last_clean":"2024-08-17T17:34:48.450903+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:34:48.450903+0000","last_undegraded":"2024-08-17T17:34:48.450903+0000","last_fullsized":"2024-08-17T17:34:48.450903+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":407,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.579214+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:34:49.579214+0000","last_peered":"2024-08-17T17:34:49.579214+0000","last_clean":"2024-08-17T17:34:49.579214+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:34:49.579214+0000","last_undegraded":"2024-08-17T17:34:49.579214+0000","last_fullsized":"2024-08-17T17:34:49.579214+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":420,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.452551+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:34:48.452551+0000","last_peered":"2024-08-17T17:34:48.452551+0000","last_clean":"2024-08-17T17:34:48.452551+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:34:48.452551+0000","last_undegraded":"2024-08-17T17:34:48.452551+0000","last_fullsized":"2024-08-17T17:34:48.452551+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":164,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577557+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:34:49.577557+0000","last_peered":"2024-08-17T17:34:49.577557+0000","last_clean":"2024-08-17T17:34:49.577557+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:34:49.577557+0000","last_undegraded":"2024-08-17T17:34:49.577557+0000","last_fullsized":"2024-08-17T17:34:49.577557+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":431,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.578414+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:34:49.578414+0000","last_peered":"2024-08-17T17:34:49.578414+0000","last_clean":"2024-08-17T17:34:49.578414+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:34:49.578414+0000","last_undegraded":"2024-08-17T17:34:49.578414+0000","last_fullsized":"2024-08-17T17:34:49.578414+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":406,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450670+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:34:48.450670+0000","last_peered":"2024-08-17T17:34:48.450670+0000","last_clean":"2024-08-17T17:34:48.450670+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:34:48.450670+0000","last_undegraded":"2024-08-17T17:34:48.450670+0000","last_fullsized":"2024-08-17T17:34:48.450670+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":164,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577718+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:34:49.577718+0000","last_peered":"2024-08-17T17:34:49.577718+0000","last_clean":"2024-08-17T17:34:49.577718+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:34:49.577718+0000","last_undegraded":"2024-08-17T17:34:49.577718+0000","last_fullsized":"2024-08-17T17:34:49.577718+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":61,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446002+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:34:48.446002+0000","last_peered":"2024-08-17T17:34:48.446002+0000","last_clean":"2024-08-17T17:34:48.446002+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:34:48.446002+0000","last_undegraded":"2024-08-17T17:34:48.446002+0000","last_fullsized":"2024-08-17T17:34:48.446002+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":164,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-17T17:34:49.577444+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:34:49.577444+0000","last_peered":"2024-08-17T17:34:49.577444+0000","last_clean":"2024-08-17T17:34:49.577444+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:34:49.577444+0000","last_undegraded":"2024-08-17T17:34:49.577444+0000","last_fullsized":"2024-08-17T17:34:49.577444+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":71,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.446179+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:34:48.446179+0000","last_peered":"2024-08-17T17:34:48.446179+0000","last_clean":"2024-08-17T17:34:48.446179+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:34:48.446179+0000","last_undegraded":"2024-08-17T17:34:48.446179+0000","last_fullsized":"2024-08-17T17:34:48.446179+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":425,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.452065+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:34:48.452065+0000","last_peered":"2024-08-17T17:34:48.452065+0000","last_clean":"2024-08-17T17:34:48.452065+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:34:48.452065+0000","last_undegraded":"2024-08-17T17:34:48.452065+0000","last_fullsized":"2024-08-17T17:34:48.452065+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":425,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.453548+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:34:48.453548+0000","last_peered":"2024-08-17T17:34:48.453548+0000","last_clean":"2024-08-17T17:34:48.453548+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:34:48.453548+0000","last_undegraded":"2024-08-17T17:34:48.453548+0000","last_fullsized":"2024-08-17T17:34:48.453548+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.450346+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:34:48.450346+0000","last_peered":"2024-08-17T17:34:48.450346+0000","last_clean":"2024-08-17T17:34:48.450346+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:34:48.450346+0000","last_undegraded":"2024-08-17T17:34:48.450346+0000","last_fullsized":"2024-08-17T17:34:48.450346+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":445,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-17T17:34:48.451564+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:34:48.451564+0000","last_peered":"2024-08-17T17:34:48.451564+0000","last_clean":"2024-08-17T17:34:48.451564+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:34:48.451564+0000","last_undegraded":"2024-08-17T17:34:48.451564+0000","last_fullsized":"2024-08-17T17:34:48.451564+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201656,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462552,"kb_used_data":8132,"kb_used_omap":1,"kb_used_meta":454398,"kb_avail":93282600,"statfs":{"total":95995035648,"available":95521382400,"internally_reserved":0,"allocated":8327168,"data_stored":6834627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":465304011},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922152,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":141688,"kb_used_data":4008,"kb_used_omap":7,"kb_used_meta":137656,"kb_avail":93603464,"statfs":{"total":95995035648,"available":95849947136,"internally_reserved":0,"allocated":4104192,"data_stored":2630686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":140960438},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085676,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":90492,"kb_used_data":3440,"kb_used_omap":7,"kb_used_meta":87032,"kb_avail":93654660,"statfs":{"total":95995035648,"available":95902371840,"internally_reserved":0,"allocated":3522560,"data_stored":2045654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":89121209},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986566,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":471816,"kb_used_data":7548,"kb_used_omap":1,"kb_used_meta":464254,"kb_avail":93273336,"statfs":{"total":95995035648,"available":95511896064,"internally_reserved":0,"allocated":7729152,"data_stored":6244197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":475396550},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435457,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522396,"kb_used_data":3408,"kb_used_omap":5,"kb_used_meta":518970,"kb_avail":93222756,"statfs":{"total":95995035648,"available":95460102144,"internally_reserved":0,"allocated":3489792,"data_stored":2028166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":531425353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598984,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602936,"kb_used_data":3432,"kb_used_omap":7,"kb_used_meta":599480,"kb_avail":93142216,"statfs":{"total":95995035648,"available":95377629184,"internally_reserved":0,"allocated":3514368,"data_stored":2044442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":613867693},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762510,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440492,"kb_used_data":3424,"kb_used_omap":7,"kb_used_meta":437048,"kb_avail":93304660,"statfs":{"total":95995035648,"available":95543971840,"internally_reserved":0,"allocated":3506176,"data_stored":2043572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":447537569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926036,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":531256,"kb_used_data":8104,"kb_used_omap":7,"kb_used_meta":523128,"kb_avail":93213896,"statfs":{"total":95995035648,"available":95451029504,"internally_reserved":0,"allocated":8298496,"data_stored":6821949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":535683676},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:34:57.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:57.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:57.568 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:57.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.591 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:57.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:57.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.718 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.842 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:57.966 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:58.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:58.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:58 smithi064 ceph-mon[104516]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:58.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:58.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:58.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:58.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:58.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:58 smithi064 ceph-mon[109246]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:58.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:58.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:58.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:58 smithi175 ceph-mon[101434]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:34:58.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:58.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:58.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:58.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.840 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:58.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:58.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:58.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:59.066 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:59.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:59.152 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:34:59.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:59.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:34:59.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:59.187 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:34:59.188 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:59.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:34:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:59.217 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:59.220 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:34:59.221 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:59.238 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:34:59.238 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:59.255 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:34:59.255 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:59.272 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:34:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:59.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.343 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:59.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:59.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:34:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1972140333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:59.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:34:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1972140333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:59.445 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:59.470 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:59.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:34:59.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:34:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1972140333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:59.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:59.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:59.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:59.724 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:34:59.724 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:34:59.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:59.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:59.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:59.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:34:59.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:00.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:00.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:00.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:00.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:00.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:00.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:00.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:00.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:35:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[104516]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2704730814' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[109246]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2704730814' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:35:00.451 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:00.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:00.533 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-08-17T17:35:00.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:00.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:00 smithi175 ceph-mon[101434]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2704730814' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:00.702 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:00.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:00.829 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:00.854 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:00.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:00.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:00.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:01.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:01.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.232 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:01.357 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:01.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:01 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:01.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:01 smithi064 ceph-mon[104516]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:35:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:01 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:01.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:01 smithi064 ceph-mon[109246]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:35:01.459 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:01.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:01.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:01.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:01.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:01.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:01 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:01 smithi175 ceph-mon[101434]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:35:01.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:01.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:01.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:01.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:01.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:01.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:02.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:02.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:02.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:02.248 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:02.282 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:02.283 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:02.283 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:02.351 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:02.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:02.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:02 smithi064 ceph-mon[104516]: pgmap v899: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:02.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:02 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:02.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:02 smithi064 ceph-mon[109246]: pgmap v899: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:02.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:02 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:02.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:02.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:02.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:02.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:02 smithi175 ceph-mon[101434]: pgmap v899: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:02 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:02.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:02.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:02.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:02.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:02.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:02.990 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-08-17T17:35:02.992 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:35:02.992 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:35:03.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:03.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:03.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:03.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:35:03.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:35:03.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_242541 2024-08-17T17:35:03.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:03.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:35:03.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:03.265 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:03.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:03.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:03.507 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:03.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:03.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:03.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:03.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:03.840 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:03.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:03.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:04.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:04.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:04.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:04.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3429 3413 213.273 213.312 0.00403476 0.00467445 2024-08-17T17:35:04.155 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:04.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:04.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:04 smithi064 ceph-mon[104516]: pgmap v900: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:04.267 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:03] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-17T17:35:04.268 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:04 smithi064 ceph-mon[109246]: pgmap v900: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:04.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:04.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:04.429 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:04.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:04.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:04.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:04 smithi175 ceph-mon[101434]: pgmap v900: 89 pgs: 89 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:35:04.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:04.704 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:04.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:04.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:04.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:04.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:05.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:05.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:05.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6880 6864 214.473 215.688 0.00360055 0.00465183 2024-08-17T17:35:05.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:05.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:05.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:05.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:05.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:05.648 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:05.750 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:05.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:05.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:05.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:06.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9926 9910 206.435 190.375 0.00333688 0.00483512 2024-08-17T17:35:06.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:06.191 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:06.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.323 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.388 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:06.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:06.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:06 smithi064 ceph-mon[104516]: pgmap v901: 89 pgs: 89 active+clean; 204 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 318 op/s 2024-08-17T17:35:06.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:06 smithi064 ceph-mon[109246]: pgmap v901: 89 pgs: 89 active+clean; 204 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 318 op/s 2024-08-17T17:35:06.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:06.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:06 smithi175 ceph-mon[101434]: pgmap v901: 89 pgs: 89 active+clean; 204 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s wr, 318 op/s 2024-08-17T17:35:06.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:06.721 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:06.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:06.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:06.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:06.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:07.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:07.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:07.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 12870 12854 200.822 184 0.00249288 0.00497406 2024-08-17T17:35:07.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:07.280 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:07.381 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:07.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:07.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:07.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:07.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:07.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:07.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:07.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:07.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:07.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:07.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:07.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:08.101 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15321 15305 191.292 153.188 0.00553598 0.00522144 2024-08-17T17:35:08.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:08.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:08.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:08.356 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:35:08.357 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:35:08.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:08.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:08 smithi064 ceph-mon[104516]: pgmap v902: 89 pgs: 89 active+clean; 379 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s wr, 598 op/s 2024-08-17T17:35:08.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:08 smithi064 ceph-mon[109246]: pgmap v902: 89 pgs: 89 active+clean; 379 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s wr, 598 op/s 2024-08-17T17:35:08.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:08.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:08 smithi175 ceph-mon[101434]: pgmap v902: 89 pgs: 89 active+clean; 379 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s wr, 598 op/s 2024-08-17T17:35:08.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:08.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:08.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:08.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:09.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:09.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:09.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17697 17681 184.158 148.5 0.0108788 0.00542374 2024-08-17T17:35:09.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:09.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:09.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:09.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:09.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:09.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:09.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:09.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:09.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:09 smithi175 ceph-mon[101434]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:09.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:09 smithi064 ceph-mon[104516]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:09 smithi064 ceph-mon[109246]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:09.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:09.817 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:09.817 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:09.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:09.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:09.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:09.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:10.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:10.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:10.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 19997 19981 178.384 143.75 0.00184025 0.00559753 2024-08-17T17:35:10.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:10.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:10.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.545 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.647 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:10 smithi175 ceph-mon[101434]: pgmap v903: 89 pgs: 89 active+clean; 553 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2024-08-17T17:35:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:10.679 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[104516]: pgmap v903: 89 pgs: 89 active+clean; 553 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2024-08-17T17:35:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[109246]: pgmap v903: 89 pgs: 89 active+clean; 553 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2024-08-17T17:35:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:10.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:10.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:10.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:10.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:11.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:11.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:11.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 22026 22010 171.936 126.812 0.00169101 0.0057714 2024-08-17T17:35:11.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:11.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:11.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:11.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:11.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:11.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:11.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:11.695 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:11.726 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:11.828 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:11.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:11.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:11.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:12.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:12.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:12.141 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 23423 23407 162.532 87.3125 0.00238203 0.00613362 2024-08-17T17:35:12.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:12.243 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:12.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:12.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:12.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:12.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:12 smithi064 ceph-mon[109246]: pgmap v904: 89 pgs: 89 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:35:12.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:12 smithi064 ceph-mon[104516]: pgmap v904: 89 pgs: 89 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:35:12.510 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:12.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:12.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:12 smithi175 ceph-mon[101434]: pgmap v904: 89 pgs: 89 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:35:12.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:12.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:12.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:12.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:12.913 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:13.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:13.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:13.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:13.142 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 25034 25018 156.347 100.688 0.0288991 0.00638204 2024-08-17T17:35:13.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:13.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:13.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:13.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:13.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:13.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:13.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:13.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:13.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:13.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:13.718 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:13.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:13.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:13.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:14.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:14.133 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:14.143 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 27343 27327 155.238 144.312 0.00481602 0.00643457 2024-08-17T17:35:14.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:35:14.226 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:13] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.51.0" 2024-08-17T17:35:14.235 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.265 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:14.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:14.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:14.524 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:14 smithi175 ceph-mon[101434]: pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:35:14.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:14 smithi064 ceph-mon[104516]: pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:35:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:14 smithi064 ceph-mon[109246]: pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:35:14.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:14.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:14.876 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:14.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.008 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:15.142 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 29480 29464 153.431 133.562 0.0052721 0.00651161 2024-08-17T17:35:15.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:15.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:15.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:15.428 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:15.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:15.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:15.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:15.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:15.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:15.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:15.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:15.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:16.041 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":902,"stamp":"2024-08-17T17:35:07.116210+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":397088741,"num_objects":6014,"num_object_clones":0,"num_object_copies":18042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6014,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":7109,"num_write_kb":398145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":10166083,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7168,"ondisk_log_size":7168,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4304780,"kb_used_data":1056648,"kb_used_omap":46,"kb_used_meta":3248017,"kb_avail":745656436,"statfs":{"total":767960285184,"available":763552190464,"internally_reserved":0,"allocated":1082007552,"data_stored":1070132554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":3325969651},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5986,"num_write_kb":383104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002776"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":413,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360679+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:35:01.360679+0000","last_peered":"2024-08-17T17:35:01.360679+0000","last_clean":"2024-08-17T17:35:01.360679+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:35:01.360679+0000","last_undegraded":"2024-08-17T17:35:01.360679+0000","last_fullsized":"2024-08-17T17:35:01.360679+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":275,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.077847+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:34:53.077847+0000","last_peered":"2024-08-17T17:34:53.077847+0000","last_clean":"2024-08-17T17:34:53.077847+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:34:53.077847+0000","last_undegraded":"2024-08-17T17:34:53.077847+0000","last_fullsized":"2024-08-17T17:34:53.077847+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617592+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:34:52.617592+0000","last_peered":"2024-08-17T17:34:52.617592+0000","last_clean":"2024-08-17T17:34:52.617592+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:34:52.617592+0000","last_undegraded":"2024-08-17T17:34:52.617592+0000","last_fullsized":"2024-08-17T17:34:52.617592+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":405,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627065+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:35:00.627065+0000","last_peered":"2024-08-17T17:35:00.627065+0000","last_clean":"2024-08-17T17:35:00.627065+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:35:00.627065+0000","last_undegraded":"2024-08-17T17:35:00.627065+0000","last_fullsized":"2024-08-17T17:35:00.627065+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":405,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627443+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:35:00.627443+0000","last_peered":"2024-08-17T17:35:00.627443+0000","last_clean":"2024-08-17T17:35:00.627443+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:35:00.627443+0000","last_undegraded":"2024-08-17T17:35:00.627443+0000","last_fullsized":"2024-08-17T17:35:00.627443+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.077960+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:34:53.077960+0000","last_peered":"2024-08-17T17:34:53.077960+0000","last_clean":"2024-08-17T17:34:53.077960+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:34:53.077960+0000","last_undegraded":"2024-08-17T17:34:53.077960+0000","last_fullsized":"2024-08-17T17:34:53.077960+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":413,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360900+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:35:01.360900+0000","last_peered":"2024-08-17T17:35:01.360900+0000","last_clean":"2024-08-17T17:35:01.360900+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:35:01.360900+0000","last_undegraded":"2024-08-17T17:35:01.360900+0000","last_fullsized":"2024-08-17T17:35:01.360900+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":209,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.077670+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:34:53.077670+0000","last_peered":"2024-08-17T17:34:53.077670+0000","last_clean":"2024-08-17T17:34:53.077670+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:34:53.077670+0000","last_undegraded":"2024-08-17T17:34:53.077670+0000","last_fullsized":"2024-08-17T17:34:53.077670+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"278'564","reported_seq":576,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:05.629115+0000","last_change":"2024-08-17T17:34:48.462975+0000","last_active":"2024-08-17T17:35:05.629115+0000","last_peered":"2024-08-17T17:35:05.629115+0000","last_clean":"2024-08-17T17:35:05.629115+0000","last_became_active":"2024-08-17T17:34:48.462675+0000","last_became_peered":"2024-08-17T17:34:48.462675+0000","last_unstale":"2024-08-17T17:35:05.629115+0000","last_undegraded":"2024-08-17T17:35:05.629115+0000","last_fullsized":"2024-08-17T17:35:05.629115+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":564,"log_dups_size":0,"ondisk_log_size":564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:53.644325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":564,"num_write_kb":36096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":271,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.198409+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:35:01.198409+0000","last_peered":"2024-08-17T17:35:01.198409+0000","last_clean":"2024-08-17T17:35:01.198409+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:35:01.198409+0000","last_undegraded":"2024-08-17T17:35:01.198409+0000","last_fullsized":"2024-08-17T17:35:01.198409+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617173+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:34:52.617173+0000","last_peered":"2024-08-17T17:34:52.617173+0000","last_clean":"2024-08-17T17:34:52.617173+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:34:52.617173+0000","last_undegraded":"2024-08-17T17:34:52.617173+0000","last_fullsized":"2024-08-17T17:34:52.617173+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"278'320","reported_seq":333,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.452348+0000","last_change":"2024-08-17T17:34:48.460978+0000","last_active":"2024-08-17T17:35:04.452348+0000","last_peered":"2024-08-17T17:35:04.452348+0000","last_clean":"2024-08-17T17:35:04.452348+0000","last_became_active":"2024-08-17T17:34:48.460684+0000","last_became_peered":"2024-08-17T17:34:48.460684+0000","last_unstale":"2024-08-17T17:35:04.452348+0000","last_undegraded":"2024-08-17T17:35:04.452348+0000","last_fullsized":"2024-08-17T17:35:04.452348+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":320,"log_dups_size":0,"ondisk_log_size":320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:51.491253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20971520,"num_objects":320,"num_object_clones":0,"num_object_copies":960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":320,"num_write_kb":20480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":272,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078293+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:34:53.078293+0000","last_peered":"2024-08-17T17:34:53.078293+0000","last_clean":"2024-08-17T17:34:53.078293+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:34:53.078293+0000","last_undegraded":"2024-08-17T17:34:53.078293+0000","last_fullsized":"2024-08-17T17:34:53.078293+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078298+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:34:53.078298+0000","last_peered":"2024-08-17T17:34:53.078298+0000","last_clean":"2024-08-17T17:34:53.078298+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:34:53.078298+0000","last_undegraded":"2024-08-17T17:34:53.078298+0000","last_fullsized":"2024-08-17T17:34:53.078298+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"278'312","reported_seq":331,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.564366+0000","last_change":"2024-08-17T17:34:48.460837+0000","last_active":"2024-08-17T17:35:04.564366+0000","last_peered":"2024-08-17T17:35:04.564366+0000","last_clean":"2024-08-17T17:35:04.564366+0000","last_became_active":"2024-08-17T17:34:48.460556+0000","last_became_peered":"2024-08-17T17:34:48.460556+0000","last_unstale":"2024-08-17T17:35:04.564366+0000","last_undegraded":"2024-08-17T17:35:04.564366+0000","last_fullsized":"2024-08-17T17:35:04.564366+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":312,"log_dups_size":0,"ondisk_log_size":312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:42.107578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20447232,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":312,"num_write_kb":19968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":289,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078395+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:34:53.078395+0000","last_peered":"2024-08-17T17:34:53.078395+0000","last_clean":"2024-08-17T17:34:53.078395+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:34:53.078395+0000","last_undegraded":"2024-08-17T17:34:53.078395+0000","last_fullsized":"2024-08-17T17:34:53.078395+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":189,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078514+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:34:53.078514+0000","last_peered":"2024-08-17T17:34:53.078514+0000","last_clean":"2024-08-17T17:34:53.078514+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:34:53.078514+0000","last_undegraded":"2024-08-17T17:34:53.078514+0000","last_fullsized":"2024-08-17T17:34:53.078514+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"278'380","reported_seq":399,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.939381+0000","last_change":"2024-08-17T17:34:48.460872+0000","last_active":"2024-08-17T17:35:04.939381+0000","last_peered":"2024-08-17T17:35:04.939381+0000","last_clean":"2024-08-17T17:35:04.939381+0000","last_became_active":"2024-08-17T17:34:48.460576+0000","last_became_peered":"2024-08-17T17:34:48.460576+0000","last_unstale":"2024-08-17T17:35:04.939381+0000","last_undegraded":"2024-08-17T17:35:04.939381+0000","last_fullsized":"2024-08-17T17:35:04.939381+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":380,"log_dups_size":0,"ondisk_log_size":380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:03.829227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24903680,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":380,"num_write_kb":24320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617517+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:34:52.617517+0000","last_peered":"2024-08-17T17:34:52.617517+0000","last_clean":"2024-08-17T17:34:52.617517+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:34:52.617517+0000","last_undegraded":"2024-08-17T17:34:52.617517+0000","last_fullsized":"2024-08-17T17:34:52.617517+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":419,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627266+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:35:00.627266+0000","last_peered":"2024-08-17T17:35:00.627266+0000","last_clean":"2024-08-17T17:35:00.627266+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:35:00.627266+0000","last_undegraded":"2024-08-17T17:35:00.627266+0000","last_fullsized":"2024-08-17T17:35:00.627266+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"278'92","reported_seq":111,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:03.604160+0000","last_change":"2024-08-17T17:34:48.458793+0000","last_active":"2024-08-17T17:35:03.604160+0000","last_peered":"2024-08-17T17:35:03.604160+0000","last_clean":"2024-08-17T17:35:03.604160+0000","last_became_active":"2024-08-17T17:34:48.458438+0000","last_became_peered":"2024-08-17T17:34:48.458438+0000","last_unstale":"2024-08-17T17:35:03.604160+0000","last_undegraded":"2024-08-17T17:35:03.604160+0000","last_fullsized":"2024-08-17T17:35:03.604160+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:58.357329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":435,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360684+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:35:01.360684+0000","last_peered":"2024-08-17T17:35:01.360684+0000","last_clean":"2024-08-17T17:35:01.360684+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:35:01.360684+0000","last_undegraded":"2024-08-17T17:35:01.360684+0000","last_fullsized":"2024-08-17T17:35:01.360684+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":181,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078905+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:34:53.078905+0000","last_peered":"2024-08-17T17:34:53.078905+0000","last_clean":"2024-08-17T17:34:53.078905+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:34:53.078905+0000","last_undegraded":"2024-08-17T17:34:53.078905+0000","last_fullsized":"2024-08-17T17:34:53.078905+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"278'446","reported_seq":465,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.961274+0000","last_change":"2024-08-17T17:34:48.463240+0000","last_active":"2024-08-17T17:35:04.961274+0000","last_peered":"2024-08-17T17:35:04.961274+0000","last_clean":"2024-08-17T17:35:04.961274+0000","last_became_active":"2024-08-17T17:34:48.462973+0000","last_became_peered":"2024-08-17T17:34:48.462973+0000","last_unstale":"2024-08-17T17:35:04.961274+0000","last_undegraded":"2024-08-17T17:35:04.961274+0000","last_fullsized":"2024-08-17T17:35:04.961274+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":446,"log_dups_size":0,"ondisk_log_size":446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:39.595929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":446,"num_write_kb":28544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":196,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078216+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:34:53.078216+0000","last_peered":"2024-08-17T17:34:53.078216+0000","last_clean":"2024-08-17T17:34:53.078216+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:34:53.078216+0000","last_undegraded":"2024-08-17T17:34:53.078216+0000","last_fullsized":"2024-08-17T17:34:53.078216+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078804+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:34:53.078804+0000","last_peered":"2024-08-17T17:34:53.078804+0000","last_clean":"2024-08-17T17:34:53.078804+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:34:53.078804+0000","last_undegraded":"2024-08-17T17:34:53.078804+0000","last_fullsized":"2024-08-17T17:34:53.078804+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"278'707","reported_seq":720,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:06.765582+0000","last_change":"2024-08-17T17:34:48.453342+0000","last_active":"2024-08-17T17:35:06.765582+0000","last_peered":"2024-08-17T17:35:06.765582+0000","last_clean":"2024-08-17T17:35:06.765582+0000","last_became_active":"2024-08-17T17:34:48.453113+0000","last_became_peered":"2024-08-17T17:34:48.453113+0000","last_unstale":"2024-08-17T17:35:06.765582+0000","last_undegraded":"2024-08-17T17:35:06.765582+0000","last_fullsized":"2024-08-17T17:35:06.765582+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":707,"log_dups_size":0,"ondisk_log_size":707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:39.280205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46333952,"num_objects":707,"num_object_clones":0,"num_object_copies":2121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":707,"num_write_kb":45248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078196+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:34:53.078196+0000","last_peered":"2024-08-17T17:34:53.078196+0000","last_clean":"2024-08-17T17:34:53.078196+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:34:53.078196+0000","last_undegraded":"2024-08-17T17:34:53.078196+0000","last_fullsized":"2024-08-17T17:34:53.078196+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078952+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:34:53.078952+0000","last_peered":"2024-08-17T17:34:53.078952+0000","last_clean":"2024-08-17T17:34:53.078952+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:34:53.078952+0000","last_undegraded":"2024-08-17T17:34:53.078952+0000","last_fullsized":"2024-08-17T17:34:53.078952+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"278'381","reported_seq":400,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.954267+0000","last_change":"2024-08-17T17:34:48.461336+0000","last_active":"2024-08-17T17:35:04.954267+0000","last_peered":"2024-08-17T17:35:04.954267+0000","last_clean":"2024-08-17T17:35:04.954267+0000","last_became_active":"2024-08-17T17:34:48.461060+0000","last_became_peered":"2024-08-17T17:34:48.461060+0000","last_unstale":"2024-08-17T17:35:04.954267+0000","last_undegraded":"2024-08-17T17:35:04.954267+0000","last_fullsized":"2024-08-17T17:35:04.954267+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":381,"log_dups_size":0,"ondisk_log_size":381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:41.170542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":381,"num_write_kb":24384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":415,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360510+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:35:01.360510+0000","last_peered":"2024-08-17T17:35:01.360510+0000","last_clean":"2024-08-17T17:35:01.360510+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:35:01.360510+0000","last_undegraded":"2024-08-17T17:35:01.360510+0000","last_fullsized":"2024-08-17T17:35:01.360510+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":403,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.735073+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:35:00.735073+0000","last_peered":"2024-08-17T17:35:00.735073+0000","last_clean":"2024-08-17T17:35:00.735073+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:35:00.735073+0000","last_undegraded":"2024-08-17T17:35:00.735073+0000","last_fullsized":"2024-08-17T17:35:00.735073+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"278'484","reported_seq":496,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:05.628967+0000","last_change":"2024-08-17T17:34:48.455278+0000","last_active":"2024-08-17T17:35:05.628967+0000","last_peered":"2024-08-17T17:35:05.628967+0000","last_clean":"2024-08-17T17:35:05.628967+0000","last_became_active":"2024-08-17T17:34:48.455018+0000","last_became_peered":"2024-08-17T17:34:48.455018+0000","last_unstale":"2024-08-17T17:35:05.628967+0000","last_undegraded":"2024-08-17T17:35:05.628967+0000","last_fullsized":"2024-08-17T17:35:05.628967+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":484,"log_dups_size":0,"ondisk_log_size":484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:53.287262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":484,"num_write_kb":30976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":591,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.077993+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:34:53.077993+0000","last_peered":"2024-08-17T17:34:53.077993+0000","last_clean":"2024-08-17T17:34:53.077993+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:34:53.077993+0000","last_undegraded":"2024-08-17T17:34:53.077993+0000","last_fullsized":"2024-08-17T17:34:53.077993+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":867,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617323+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:34:52.617323+0000","last_peered":"2024-08-17T17:34:52.617323+0000","last_clean":"2024-08-17T17:34:52.617323+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:34:52.617323+0000","last_undegraded":"2024-08-17T17:34:52.617323+0000","last_fullsized":"2024-08-17T17:34:52.617323+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":398,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360349+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:35:01.360349+0000","last_peered":"2024-08-17T17:35:01.360349+0000","last_clean":"2024-08-17T17:35:01.360349+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:35:01.360349+0000","last_undegraded":"2024-08-17T17:35:01.360349+0000","last_fullsized":"2024-08-17T17:35:01.360349+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":180,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078663+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:34:53.078663+0000","last_peered":"2024-08-17T17:34:53.078663+0000","last_clean":"2024-08-17T17:34:53.078663+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:34:53.078663+0000","last_undegraded":"2024-08-17T17:34:53.078663+0000","last_fullsized":"2024-08-17T17:34:53.078663+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"278'448","reported_seq":467,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.955639+0000","last_change":"2024-08-17T17:34:48.462773+0000","last_active":"2024-08-17T17:35:04.955639+0000","last_peered":"2024-08-17T17:35:04.955639+0000","last_clean":"2024-08-17T17:35:04.955639+0000","last_became_active":"2024-08-17T17:34:48.462475+0000","last_became_peered":"2024-08-17T17:34:48.462475+0000","last_unstale":"2024-08-17T17:35:04.955639+0000","last_undegraded":"2024-08-17T17:35:04.955639+0000","last_fullsized":"2024-08-17T17:35:04.955639+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":448,"log_dups_size":0,"ondisk_log_size":448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:09.376736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":448,"num_write_kb":28672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":654,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078003+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:34:53.078003+0000","last_peered":"2024-08-17T17:34:53.078003+0000","last_clean":"2024-08-17T17:34:53.078003+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:34:53.078003+0000","last_undegraded":"2024-08-17T17:34:53.078003+0000","last_fullsized":"2024-08-17T17:34:53.078003+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":420,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627029+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:35:00.627029+0000","last_peered":"2024-08-17T17:35:00.627029+0000","last_clean":"2024-08-17T17:35:00.627029+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:35:00.627029+0000","last_undegraded":"2024-08-17T17:35:00.627029+0000","last_fullsized":"2024-08-17T17:35:00.627029+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":417,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627508+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:35:00.627508+0000","last_peered":"2024-08-17T17:35:00.627508+0000","last_clean":"2024-08-17T17:35:00.627508+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:35:00.627508+0000","last_undegraded":"2024-08-17T17:35:00.627508+0000","last_fullsized":"2024-08-17T17:35:00.627508+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"278'101","reported_seq":120,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:03.608878+0000","last_change":"2024-08-17T17:34:48.457912+0000","last_active":"2024-08-17T17:35:03.608878+0000","last_peered":"2024-08-17T17:35:03.608878+0000","last_clean":"2024-08-17T17:35:03.608878+0000","last_became_active":"2024-08-17T17:34:48.457628+0000","last_became_peered":"2024-08-17T17:34:48.457628+0000","last_unstale":"2024-08-17T17:35:03.608878+0000","last_undegraded":"2024-08-17T17:35:03.608878+0000","last_fullsized":"2024-08-17T17:35:03.608878+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":101,"log_dups_size":0,"ondisk_log_size":101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:52.090342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6619136,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":6464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":818,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.734736+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:35:00.734736+0000","last_peered":"2024-08-17T17:35:00.734736+0000","last_clean":"2024-08-17T17:35:00.734736+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:35:00.734736+0000","last_undegraded":"2024-08-17T17:35:00.734736+0000","last_fullsized":"2024-08-17T17:35:00.734736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":409,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.187869+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:35:00.187869+0000","last_peered":"2024-08-17T17:35:00.187869+0000","last_clean":"2024-08-17T17:35:00.187869+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:35:00.187869+0000","last_undegraded":"2024-08-17T17:35:00.187869+0000","last_fullsized":"2024-08-17T17:35:00.187869+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":456,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.188551+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:35:00.188551+0000","last_peered":"2024-08-17T17:35:00.188551+0000","last_clean":"2024-08-17T17:35:00.188551+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:35:00.188551+0000","last_undegraded":"2024-08-17T17:35:00.188551+0000","last_fullsized":"2024-08-17T17:35:00.188551+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"278'521","reported_seq":533,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:05.629270+0000","last_change":"2024-08-17T17:34:48.462500+0000","last_active":"2024-08-17T17:35:05.629270+0000","last_peered":"2024-08-17T17:35:05.629270+0000","last_clean":"2024-08-17T17:35:05.629270+0000","last_became_active":"2024-08-17T17:34:48.462215+0000","last_became_peered":"2024-08-17T17:34:48.462215+0000","last_unstale":"2024-08-17T17:35:05.629270+0000","last_undegraded":"2024-08-17T17:35:05.629270+0000","last_fullsized":"2024-08-17T17:35:05.629270+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":521,"log_dups_size":0,"ondisk_log_size":521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:31:21.306309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144256,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":521,"num_write_kb":33344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":990,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617510+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:34:52.617510+0000","last_peered":"2024-08-17T17:34:52.617510+0000","last_clean":"2024-08-17T17:34:52.617510+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:34:52.617510+0000","last_undegraded":"2024-08-17T17:34:52.617510+0000","last_fullsized":"2024-08-17T17:34:52.617510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":235,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.734780+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:35:00.734780+0000","last_peered":"2024-08-17T17:35:00.734780+0000","last_clean":"2024-08-17T17:35:00.734780+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:35:00.734780+0000","last_undegraded":"2024-08-17T17:35:00.734780+0000","last_fullsized":"2024-08-17T17:35:00.734780+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":303,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.187988+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:35:00.187988+0000","last_peered":"2024-08-17T17:35:00.187988+0000","last_clean":"2024-08-17T17:35:00.187988+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:35:00.187988+0000","last_undegraded":"2024-08-17T17:35:00.187988+0000","last_fullsized":"2024-08-17T17:35:00.187988+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"278'292","reported_seq":311,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.557941+0000","last_change":"2024-08-17T17:34:48.464724+0000","last_active":"2024-08-17T17:35:04.557941+0000","last_peered":"2024-08-17T17:35:04.557941+0000","last_clean":"2024-08-17T17:35:04.557941+0000","last_became_active":"2024-08-17T17:34:48.464408+0000","last_became_peered":"2024-08-17T17:34:48.464408+0000","last_unstale":"2024-08-17T17:35:04.557941+0000","last_undegraded":"2024-08-17T17:35:04.557941+0000","last_fullsized":"2024-08-17T17:35:04.557941+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":292,"log_dups_size":0,"ondisk_log_size":292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:22:49.210052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":292,"num_write_kb":18688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":59,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617555+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:34:52.617555+0000","last_peered":"2024-08-17T17:34:52.617555+0000","last_clean":"2024-08-17T17:34:52.617555+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:34:52.617555+0000","last_undegraded":"2024-08-17T17:34:52.617555+0000","last_fullsized":"2024-08-17T17:34:52.617555+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":702,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.188299+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:35:00.188299+0000","last_peered":"2024-08-17T17:35:00.188299+0000","last_clean":"2024-08-17T17:35:00.188299+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:35:00.188299+0000","last_undegraded":"2024-08-17T17:35:00.188299+0000","last_fullsized":"2024-08-17T17:35:00.188299+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":426,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.188098+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:35:00.188098+0000","last_peered":"2024-08-17T17:35:00.188098+0000","last_clean":"2024-08-17T17:35:00.188098+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:35:00.188098+0000","last_undegraded":"2024-08-17T17:35:00.188098+0000","last_fullsized":"2024-08-17T17:35:00.188098+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"278'111","reported_seq":130,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:03.600630+0000","last_change":"2024-08-17T17:34:48.463247+0000","last_active":"2024-08-17T17:35:03.600630+0000","last_peered":"2024-08-17T17:35:03.600630+0000","last_clean":"2024-08-17T17:35:03.600630+0000","last_became_active":"2024-08-17T17:34:48.462966+0000","last_became_peered":"2024-08-17T17:34:48.462966+0000","last_unstale":"2024-08-17T17:35:03.600630+0000","last_undegraded":"2024-08-17T17:35:03.600630+0000","last_fullsized":"2024-08-17T17:35:03.600630+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:28.325572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7274496,"num_objects":111,"num_object_clones":0,"num_object_copies":333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":111,"num_write_kb":7104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":423,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.188268+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:35:00.188268+0000","last_peered":"2024-08-17T17:35:00.188268+0000","last_clean":"2024-08-17T17:35:00.188268+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:35:00.188268+0000","last_undegraded":"2024-08-17T17:35:00.188268+0000","last_fullsized":"2024-08-17T17:35:00.188268+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":843,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078436+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:34:53.078436+0000","last_peered":"2024-08-17T17:34:53.078436+0000","last_clean":"2024-08-17T17:34:53.078436+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:34:53.078436+0000","last_undegraded":"2024-08-17T17:34:53.078436+0000","last_fullsized":"2024-08-17T17:34:53.078436+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":420,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627139+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:35:00.627139+0000","last_peered":"2024-08-17T17:35:00.627139+0000","last_clean":"2024-08-17T17:35:00.627139+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:35:00.627139+0000","last_undegraded":"2024-08-17T17:35:00.627139+0000","last_fullsized":"2024-08-17T17:35:00.627139+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"278'303","reported_seq":320,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:04.553080+0000","last_change":"2024-08-17T17:34:48.462499+0000","last_active":"2024-08-17T17:35:04.553080+0000","last_peered":"2024-08-17T17:35:04.553080+0000","last_clean":"2024-08-17T17:35:04.553080+0000","last_became_active":"2024-08-17T17:34:48.461996+0000","last_became_peered":"2024-08-17T17:34:48.461996+0000","last_unstale":"2024-08-17T17:35:04.553080+0000","last_undegraded":"2024-08-17T17:35:04.553080+0000","last_fullsized":"2024-08-17T17:35:04.553080+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":303,"log_dups_size":0,"ondisk_log_size":303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:12.885947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":303,"num_write_kb":19392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":413,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.361016+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:35:01.361016+0000","last_peered":"2024-08-17T17:35:01.361016+0000","last_clean":"2024-08-17T17:35:01.361016+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:35:01.361016+0000","last_undegraded":"2024-08-17T17:35:01.361016+0000","last_fullsized":"2024-08-17T17:35:01.361016+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":932,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.188582+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:35:00.188582+0000","last_peered":"2024-08-17T17:35:00.188582+0000","last_clean":"2024-08-17T17:35:00.188582+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:35:00.188582+0000","last_undegraded":"2024-08-17T17:35:00.188582+0000","last_fullsized":"2024-08-17T17:35:00.188582+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":411,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360415+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:35:01.360415+0000","last_peered":"2024-08-17T17:35:01.360415+0000","last_clean":"2024-08-17T17:35:01.360415+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:35:01.360415+0000","last_undegraded":"2024-08-17T17:35:01.360415+0000","last_fullsized":"2024-08-17T17:35:01.360415+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"278'524","reported_seq":537,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:05.621824+0000","last_change":"2024-08-17T17:34:48.463451+0000","last_active":"2024-08-17T17:35:05.621824+0000","last_peered":"2024-08-17T17:35:05.621824+0000","last_clean":"2024-08-17T17:35:05.621824+0000","last_became_active":"2024-08-17T17:34:48.463124+0000","last_became_peered":"2024-08-17T17:34:48.463124+0000","last_unstale":"2024-08-17T17:35:05.621824+0000","last_undegraded":"2024-08-17T17:35:05.621824+0000","last_fullsized":"2024-08-17T17:35:05.621824+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":524,"log_dups_size":0,"ondisk_log_size":524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:58:26.982388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":524,"num_write_kb":33536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":432,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627406+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:35:00.627406+0000","last_peered":"2024-08-17T17:35:00.627406+0000","last_clean":"2024-08-17T17:35:00.627406+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:35:00.627406+0000","last_undegraded":"2024-08-17T17:35:00.627406+0000","last_fullsized":"2024-08-17T17:35:00.627406+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":399,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078093+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:34:53.078093+0000","last_peered":"2024-08-17T17:34:53.078093+0000","last_clean":"2024-08-17T17:34:53.078093+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:34:53.078093+0000","last_undegraded":"2024-08-17T17:34:53.078093+0000","last_fullsized":"2024-08-17T17:34:53.078093+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078381+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:34:53.078381+0000","last_peered":"2024-08-17T17:34:53.078381+0000","last_clean":"2024-08-17T17:34:53.078381+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:34:53.078381+0000","last_undegraded":"2024-08-17T17:34:53.078381+0000","last_fullsized":"2024-08-17T17:34:53.078381+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":181,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078920+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:34:53.078920+0000","last_peered":"2024-08-17T17:34:53.078920+0000","last_clean":"2024-08-17T17:34:53.078920+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:34:53.078920+0000","last_undegraded":"2024-08-17T17:34:53.078920+0000","last_fullsized":"2024-08-17T17:34:53.078920+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":191,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078807+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:34:53.078807+0000","last_peered":"2024-08-17T17:34:53.078807+0000","last_clean":"2024-08-17T17:34:53.078807+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:34:53.078807+0000","last_undegraded":"2024-08-17T17:34:53.078807+0000","last_fullsized":"2024-08-17T17:34:53.078807+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":408,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.187925+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:35:00.187925+0000","last_peered":"2024-08-17T17:35:00.187925+0000","last_clean":"2024-08-17T17:35:00.187925+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:35:00.187925+0000","last_undegraded":"2024-08-17T17:35:00.187925+0000","last_fullsized":"2024-08-17T17:35:00.187925+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":276,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078292+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:34:53.078292+0000","last_peered":"2024-08-17T17:34:53.078292+0000","last_clean":"2024-08-17T17:34:53.078292+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:34:53.078292+0000","last_undegraded":"2024-08-17T17:34:53.078292+0000","last_fullsized":"2024-08-17T17:34:53.078292+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617470+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:34:52.617470+0000","last_peered":"2024-08-17T17:34:52.617470+0000","last_clean":"2024-08-17T17:34:52.617470+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:34:52.617470+0000","last_undegraded":"2024-08-17T17:34:52.617470+0000","last_fullsized":"2024-08-17T17:34:52.617470+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":375,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360288+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:35:01.360288+0000","last_peered":"2024-08-17T17:35:01.360288+0000","last_clean":"2024-08-17T17:35:01.360288+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:35:01.360288+0000","last_undegraded":"2024-08-17T17:35:01.360288+0000","last_fullsized":"2024-08-17T17:35:01.360288+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":434,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627391+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:35:00.627391+0000","last_peered":"2024-08-17T17:35:00.627391+0000","last_clean":"2024-08-17T17:35:00.627391+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:35:00.627391+0000","last_undegraded":"2024-08-17T17:35:00.627391+0000","last_fullsized":"2024-08-17T17:35:00.627391+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":424,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360312+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:35:01.360312+0000","last_peered":"2024-08-17T17:35:01.360312+0000","last_clean":"2024-08-17T17:35:01.360312+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:35:01.360312+0000","last_undegraded":"2024-08-17T17:35:01.360312+0000","last_fullsized":"2024-08-17T17:35:01.360312+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617573+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:34:52.617573+0000","last_peered":"2024-08-17T17:34:52.617573+0000","last_clean":"2024-08-17T17:34:52.617573+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:34:52.617573+0000","last_undegraded":"2024-08-17T17:34:52.617573+0000","last_fullsized":"2024-08-17T17:34:52.617573+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":429,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360425+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:35:01.360425+0000","last_peered":"2024-08-17T17:35:01.360425+0000","last_clean":"2024-08-17T17:35:01.360425+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:35:01.360425+0000","last_undegraded":"2024-08-17T17:35:01.360425+0000","last_fullsized":"2024-08-17T17:35:01.360425+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":406,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627507+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:35:00.627507+0000","last_peered":"2024-08-17T17:35:00.627507+0000","last_clean":"2024-08-17T17:35:00.627507+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:35:00.627507+0000","last_undegraded":"2024-08-17T17:35:00.627507+0000","last_fullsized":"2024-08-17T17:35:00.627507+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":411,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.187764+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:35:00.187764+0000","last_peered":"2024-08-17T17:35:00.187764+0000","last_clean":"2024-08-17T17:35:00.187764+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:35:00.187764+0000","last_undegraded":"2024-08-17T17:35:00.187764+0000","last_fullsized":"2024-08-17T17:35:00.187764+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":426,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360595+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:35:01.360595+0000","last_peered":"2024-08-17T17:35:01.360595+0000","last_clean":"2024-08-17T17:35:01.360595+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:35:01.360595+0000","last_undegraded":"2024-08-17T17:35:01.360595+0000","last_fullsized":"2024-08-17T17:35:01.360595+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078210+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:34:53.078210+0000","last_peered":"2024-08-17T17:34:53.078210+0000","last_clean":"2024-08-17T17:34:53.078210+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:34:53.078210+0000","last_undegraded":"2024-08-17T17:34:53.078210+0000","last_fullsized":"2024-08-17T17:34:53.078210+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":435,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.734893+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:35:00.734893+0000","last_peered":"2024-08-17T17:35:00.734893+0000","last_clean":"2024-08-17T17:35:00.734893+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:35:00.734893+0000","last_undegraded":"2024-08-17T17:35:00.734893+0000","last_fullsized":"2024-08-17T17:35:00.734893+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":412,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627101+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:35:00.627101+0000","last_peered":"2024-08-17T17:35:00.627101+0000","last_clean":"2024-08-17T17:35:00.627101+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:35:00.627101+0000","last_undegraded":"2024-08-17T17:35:00.627101+0000","last_fullsized":"2024-08-17T17:35:00.627101+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078141+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:34:53.078141+0000","last_peered":"2024-08-17T17:34:53.078141+0000","last_clean":"2024-08-17T17:34:53.078141+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:34:53.078141+0000","last_undegraded":"2024-08-17T17:34:53.078141+0000","last_fullsized":"2024-08-17T17:34:53.078141+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":63,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617416+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:34:52.617416+0000","last_peered":"2024-08-17T17:34:52.617416+0000","last_clean":"2024-08-17T17:34:52.617416+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:34:52.617416+0000","last_undegraded":"2024-08-17T17:34:52.617416+0000","last_fullsized":"2024-08-17T17:34:52.617416+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078127+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:34:53.078127+0000","last_peered":"2024-08-17T17:34:53.078127+0000","last_clean":"2024-08-17T17:34:53.078127+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:34:53.078127+0000","last_undegraded":"2024-08-17T17:34:53.078127+0000","last_fullsized":"2024-08-17T17:34:53.078127+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":73,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:52.617528+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:34:52.617528+0000","last_peered":"2024-08-17T17:34:52.617528+0000","last_clean":"2024-08-17T17:34:52.617528+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:34:52.617528+0000","last_undegraded":"2024-08-17T17:34:52.617528+0000","last_fullsized":"2024-08-17T17:34:52.617528+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":431,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360426+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:35:01.360426+0000","last_peered":"2024-08-17T17:35:01.360426+0000","last_clean":"2024-08-17T17:35:01.360426+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:35:01.360426+0000","last_undegraded":"2024-08-17T17:35:01.360426+0000","last_fullsized":"2024-08-17T17:35:01.360426+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":432,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:01.360763+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:35:01.360763+0000","last_peered":"2024-08-17T17:35:01.360763+0000","last_clean":"2024-08-17T17:35:01.360763+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:35:01.360763+0000","last_undegraded":"2024-08-17T17:35:01.360763+0000","last_fullsized":"2024-08-17T17:35:01.360763+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":181,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-17T17:34:53.078388+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:34:53.078388+0000","last_peered":"2024-08-17T17:34:53.078388+0000","last_clean":"2024-08-17T17:34:53.078388+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:34:53.078388+0000","last_undegraded":"2024-08-17T17:34:53.078388+0000","last_fullsized":"2024-08-17T17:34:53.078388+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":451,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-17T17:35:00.627285+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:35:00.627285+0000","last_peered":"2024-08-17T17:35:00.627285+0000","last_clean":"2024-08-17T17:35:00.627285+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:35:00.627285+0000","last_undegraded":"2024-08-17T17:35:00.627285+0000","last_fullsized":"2024-08-17T17:35:00.627285+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5986,"num_write_kb":383104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1403453440,"data_stored":1403453440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5986,"ondisk_log_size":5986,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201659,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633700,"kb_used_data":183388,"kb_used_omap":1,"kb_used_meta":450302,"kb_avail":93111452,"statfs":{"total":95995035648,"available":95346126848,"internally_reserved":0,"allocated":187789312,"data_stored":186289100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":461109707},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922155,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":222932,"kb_used_data":81164,"kb_used_omap":7,"kb_used_meta":141752,"kb_avail":93522220,"statfs":{"total":95995035648,"available":95766753280,"internally_reserved":0,"allocated":83111936,"data_stored":81627074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":145154742},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085679,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285712,"kb_used_data":186376,"kb_used_omap":7,"kb_used_meta":99320,"kb_avail":93459440,"statfs":{"total":95995035648,"available":95702466560,"internally_reserved":0,"allocated":190849024,"data_stored":189364447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":101704121},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986569,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661980,"kb_used_data":189524,"kb_used_omap":1,"kb_used_meta":472446,"kb_avail":93083172,"statfs":{"total":95995035648,"available":95317168128,"internally_reserved":0,"allocated":194072576,"data_stored":192579950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":483785158},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435460,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522444,"kb_used_data":3456,"kb_used_omap":5,"kb_used_meta":518970,"kb_avail":93222708,"statfs":{"total":95995035648,"available":95460052992,"internally_reserved":0,"allocated":3538944,"data_stored":2061998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":531425353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598987,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775944,"kb_used_data":166720,"kb_used_omap":7,"kb_used_meta":609208,"kb_avail":92969208,"statfs":{"total":95995035648,"available":95200468992,"internally_reserved":0,"allocated":170721280,"data_stored":169243683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":623829165},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762513,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":584192,"kb_used_data":155448,"kb_used_omap":7,"kb_used_meta":428728,"kb_avail":93160960,"statfs":{"total":95995035648,"available":95396823040,"internally_reserved":0,"allocated":159178752,"data_stored":157708477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":439017889},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926039,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617876,"kb_used_data":90572,"kb_used_omap":7,"kb_used_meta":527288,"kb_avail":93127276,"statfs":{"total":95995035648,"available":95362330624,"internally_reserved":0,"allocated":92745728,"data_stored":91257825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":539943516},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":119734272,"data_stored":119734272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":185466880,"data_stored":185466880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":244318208,"data_stored":244318208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":37093376,"data_stored":37093376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":240517120,"data_stored":240517120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":275251200,"data_stored":275251200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":91488256,"data_stored":91488256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":209584128,"data_stored":209584128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:35:16.045 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:35:16.045 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:35:16.046 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-08-17T17:35:16.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-08-17T17:35:16.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:16.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:16.143 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 31624 31608 151.935 134 0.0154053 0.00657628 2024-08-17T17:35:16.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:16.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:16.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:16.390 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:16.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:16 smithi064 ceph-mon[104516]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:35:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:16 smithi064 ceph-mon[109246]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:35:16.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:16.538 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:16.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:16.640 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:16 smithi175 ceph-mon[101434]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:35:16.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:16.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:16.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:16.899 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:16.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:17.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:17.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:17.143 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 33663 33647 150.184 127.438 0.00409923 0.00665254 2024-08-17T17:35:17.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:17.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:17.275 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:35:17.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:17.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:17.467 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:17.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:17.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:17 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1811541957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:17 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1811541957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:17.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:17.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:17 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1811541957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:17.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:17.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:17.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:17.983 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:17.993 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:17.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:18.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:18.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 35658 35642 148.462 124.688 0.0045604 0.00673037 2024-08-17T17:35:18.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.156 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:18.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:18.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:18.293 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:18.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:18.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:18.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[104516]: pgmap v907: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:35:18.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:18.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[104516]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[109246]: pgmap v907: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:18.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:18 smithi064 ceph-mon[109246]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:35:18.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:18 smithi175 ceph-mon[101434]: pgmap v907: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:35:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:18 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:18 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:35:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:18 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:18 smithi175 ceph-mon[101434]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:35:18.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:18.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.796 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:18.826 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.974 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:18.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:19.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:19.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:19.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 37667 37651 147.031 125.562 0.00487528 0.00679541 2024-08-17T17:35:19.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:19.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:19.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:19.390 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2626302274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2626302274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.440 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:19.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:19.442 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:19.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:19.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:19.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2626302274' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:19.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:19.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:19.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:19.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:35:20.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:20.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 39545 39529 145.286 117.375 0.00673307 0.00687823 2024-08-17T17:35:20.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:20.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:20.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:20.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:20.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:20.576 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:20 smithi175 ceph-mon[101434]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:35:20.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:20 smithi064 ceph-mon[104516]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:35:20.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:20 smithi064 ceph-mon[109246]: pgmap v909: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-17T17:35:20.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:20.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:20.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:20.871 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:20.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:21.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.015 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:21.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:21.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:21.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 41197 41181 142.946 103.25 0.00564984 0.00699093 2024-08-17T17:35:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:21.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:21.279 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:21.314 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:21.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:21.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:21.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:21.590 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:21.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:21.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:21.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:21.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:21.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:21.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:22.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:22.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:22.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:22.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 43024 43008 141.433 114.188 0.00310196 0.00706502 2024-08-17T17:35:22.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:22.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:22.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:22.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:22.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:22.508 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:22.536 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:22.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:22 smithi175 ceph-mon[101434]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-08-17T17:35:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:22 smithi064 ceph-mon[104516]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-08-17T17:35:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:22 smithi064 ceph-mon[109246]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-08-17T17:35:22.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:22.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:22.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:23.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:23.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:35:23.146276+0000 min lat: 0.00104069 max lat: 0.198104 avg lat: 0.00713382 2024-08-17T17:35:23.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:23.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 44849 44833 140.064 114.062 0.0154119 0.00713382 2024-08-17T17:35:23.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:23.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:23.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:23.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:23.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:23.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:23.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:23.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:23.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:23.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:23.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:23.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:24.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:24.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:24.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 46400 46384 138.01 96.9375 0.00300638 0.0072406 2024-08-17T17:35:24.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:23] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.51.0" 2024-08-17T17:35:24.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:24.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:24.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:24.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:24.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:24.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:24 smithi064 ceph-mon[104516]: pgmap v911: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:35:24.584 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:24 smithi064 ceph-mon[109246]: pgmap v911: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:35:24.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:24 smithi175 ceph-mon[101434]: pgmap v911: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:35:24.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:24.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:24.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:24.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:24.936 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:25.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-08-17T17:35:25.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:25.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:25.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 48152 48136 136.714 109.5 0.00683798 0.00730922 2024-08-17T17:35:25.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:25.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:25.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:25.407 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:25.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:25.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:25.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:25.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:25.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:25.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:25.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:25.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:26.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:26.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:26.161 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 49843 49827 135.276 105.688 0.00491678 0.00738568 2024-08-17T17:35:26.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:26.241 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:26.343 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:26.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:26 smithi175 ceph-mon[101434]: pgmap v912: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:35:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:26.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[104516]: pgmap v912: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:35:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:26.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:26 smithi064 ceph-mon[109246]: pgmap v912: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:35:26.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:26.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:26.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:26.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:26.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:26.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:26.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:26.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:26.985 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:27.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:27.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:27.162 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 51341 51325 133.542 93.625 0.00930282 0.00748299 2024-08-17T17:35:27.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:27.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:27.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:27.275 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:35:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70096 6.11 6.2cfda691 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e181)", 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 512.04618097399998, 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98497944299999995, 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70096 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223801+0000", 2024-08-17T17:35:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223802+0000", 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.381e-06 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223826+0000", 2024-08-17T17:35:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3985000000000002e-05 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223836+0000", 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5440000000000004e-06 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.223842+0000", 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:35:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.224625+0000", 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00078297999999999996 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.224631+0000", 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559999999999997e-06 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208765+0000", 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98413317700000003 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208781+0000", 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.592e-05 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:70132 6.11 6.2d46dcd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e181)", 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 512.00891894999995, 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94775391899999994, 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 70132 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261063+0000", 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261065+0000", 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579999999999999e-06 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261099+0000", 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4001000000000002e-05 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261122+0000", 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2575999999999999e-05 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.261133+0000", 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0815999999999999e-05 2024-08-17T17:35:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.262936+0000", 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001802943 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:55.262944+0000", 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8400000000000001e-06 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208810+0000", 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94586556700000002 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:56.208817+0000", 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0180000000000004e-06 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71363 6.13 6.18c43493 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 507.99231378100001, 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86920691299999997, 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71363 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277668+0000", 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277669+0000", 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.049e-06 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277719+0000", 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9466999999999999e-05 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277729+0000", 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8900000000000002e-06 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.277735+0000", 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5549999999999999e-06 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.278009+0000", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000273873 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.278017+0000", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146862+0000", 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86884535299999999 2024-08-17T17:35:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146875+0000", 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3327e-05 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71379 6.13 6.a90a54b3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 507.97804852000002, 2024-08-17T17:35:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85498798300000001, 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71379 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.291934+0000", 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0500000000000003e-07 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292001+0000", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7251999999999999e-05 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292012+0000", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0624e-05 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292019+0000", 2024-08-17T17:35:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5010000000000001e-06 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292064+0000", 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5040000000000002e-05 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.292069+0000", 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4299999999999997e-06 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146913+0000", 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85484384199999996 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146922+0000", 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6889999999999993e-06 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71386 6.13 6.fb4201f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 507.97067340000001, 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.847643182, 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71386 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299309+0000", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299310+0000", 2024-08-17T17:35:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1400000000000001e-06 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299334+0000", 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4596e-05 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299344+0000", 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6299999999999993e-06 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.299350+0000", 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.300057+0000", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070641699999999998 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.300065+0000", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4710000000000007e-06 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146945+0000", 2024-08-17T17:35:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.846879402 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146952+0000", 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1210000000000001e-06 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:71399 6.13 6.336b4e73 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e185)", 2024-08-17T17:35:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 507.96194357899998, 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83894056100000003, 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 71399 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308038+0000", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308040+0000", 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0389999999999999e-06 2024-08-17T17:35:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308093+0000", 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3161000000000001e-05 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308105+0000", 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2031000000000001e-05 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308112+0000", 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308165+0000", 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2988999999999997e-05 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:26:59.308172+0000", 2024-08-17T17:35:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8460000000000002e-06 2024-08-17T17:35:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146973+0000", 2024-08-17T17:35:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83880076199999998 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:00.146979+0000", 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1330000000000004e-06 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101312 6.c 6:30f79fae:::benchmark_data_smithi064_185391_object101311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:35:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 472.094025203, 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0150590660000001, 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101312 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175957+0000", 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175958+0000", 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243e-06 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175982+0000", 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3527e-05 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.175992+0000", 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0873000000000001e-05 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176000+0000", 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176053+0000", 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3437999999999999e-05 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.176059+0000", 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7690000000000002e-06 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169698+0000", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99363913199999998 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169828+0000", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012921800000000001 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169985+0000", 2024-08-17T17:35:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157904 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190558+0000", 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020572724000000001 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190868+0000", 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031017000000000002 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190931+0000", 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2501000000000003e-05 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191016+0000", 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5037999999999995e-05 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101316 6.c 6:3bdd1909:::benchmark_data_smithi064_185391_object101315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 472.091241137, 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0124066309999999, 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101316 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178741+0000", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178742+0000", 2024-08-17T17:35:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1340000000000001e-06 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178764+0000", 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2075000000000001e-05 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178776+0000", 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1766999999999999e-05 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178783+0000", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7510000000000002e-06 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178897+0000", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114405 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.178905+0000", 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9510000000000006e-06 2024-08-17T17:35:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175170+0000", 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99626462900000001 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175365+0000", 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195873 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.175512+0000", 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014636 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190619+0000", 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015107318999999999 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191035+0000", 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000416177 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191076+0000", 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0827000000000001e-05 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191148+0000", 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1363000000000001e-05 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101345 6.c 6:3c451189:::benchmark_data_smithi064_185391_object101344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 472.059412108, 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98069468500000001, 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101345 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210570+0000", 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210571+0000", 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.212e-06 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210601+0000", 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0018999999999998e-05 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210619+0000", 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7586999999999999e-05 2024-08-17T17:35:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210626+0000", 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9369999999999998e-06 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210696+0000", 2024-08-17T17:35:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0623000000000005e-05 2024-08-17T17:35:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.210705+0000", 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1479999999999995e-06 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178984+0000", 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96827804699999998 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.179094+0000", 2024-08-17T17:35:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110531 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.179234+0000", 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014043899999999999 2024-08-17T17:35:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190670+0000", 2024-08-17T17:35:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01143533 2024-08-17T17:35:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191155+0000", 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048500700000000003 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191194+0000", 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9084000000000001e-05 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191265+0000", 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0721000000000003e-05 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101356 6.c 6:30433116:::benchmark_data_smithi064_185391_object101355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:35:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:35:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 472.04370436400001, 2024-08-17T17:35:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96510339099999998, 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101356 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:35:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226278+0000", 2024-08-17T17:35:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226279+0000", 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3e-06 2024-08-17T17:35:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226302+0000", 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2875e-05 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226316+0000", 2024-08-17T17:35:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4109e-05 2024-08-17T17:35:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226324+0000", 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5519999999999999e-06 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226388+0000", 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4151000000000003e-05 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.226397+0000", 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.533e-06 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183269+0000", 2024-08-17T17:35:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95687179300000003 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183388+0000", 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011864199999999999 2024-08-17T17:35:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183529+0000", 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014124299999999999 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190713+0000", 2024-08-17T17:35:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071838409999999998 2024-08-17T17:35:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191271+0000", 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055812600000000004 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191310+0000", 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9085000000000003e-05 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191381+0000", 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1141000000000002e-05 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101360 6.c 6:3d93d054:::benchmark_data_smithi064_185391_object101359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:35:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:35:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 472.038891503, 2024-08-17T17:35:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96101782800000002, 2024-08-17T17:35:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:35:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101360 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231091+0000", 2024-08-17T17:35:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231092+0000", 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173e-06 2024-08-17T17:35:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231111+0000", 2024-08-17T17:35:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9038999999999998e-05 2024-08-17T17:35:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231122+0000", 2024-08-17T17:35:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1586000000000001e-05 2024-08-17T17:35:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231132+0000", 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2539999999999998e-06 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231190+0000", 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8338000000000003e-05 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.231198+0000", 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.678e-06 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.186989+0000", 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95579149600000002 2024-08-17T17:35:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187130+0000", 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014124799999999999 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187293+0000", 2024-08-17T17:35:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016258700000000001 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.190766+0000", 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003472709 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.191911+0000", 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011457920000000001 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.192030+0000", 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118986 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.192108+0000", 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7941999999999993e-05 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96080 7.c 7:3f226a37:::benchmark_data_smithi064_210574_object96079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:35:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 232.12442933, 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2216669489999998, 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96080 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145554+0000", 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2600000000000001e-07 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145578+0000", 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4579999999999998e-05 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145587+0000", 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5979999999999997e-06 2024-08-17T17:35:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145593+0000", 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7620000000000001e-06 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145700+0000", 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010727500000000001 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145831+0000", 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013128200000000001 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:35:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145967+0000", 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013590600000000001 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163850+0000", 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017883087999999998 2024-08-17T17:35:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163863+0000", 2024-08-17T17:35:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3019e-05 2024-08-17T17:35:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149735+0000", 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98587233299999999 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149749+0000", 2024-08-17T17:35:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3543999999999999e-05 2024-08-17T17:35:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367152+0000", 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217402664 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367220+0000", 2024-08-17T17:35:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7972000000000006e-05 2024-08-17T17:35:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96087 7.c 7:347c5818:::benchmark_data_smithi064_210574_object96086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:35:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:35:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 232.11891959799999, 2024-08-17T17:35:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.232266434, 2024-08-17T17:35:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96087 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151064+0000", 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6819999999999999e-06 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151095+0000", 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0735999999999998e-05 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151106+0000", 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1232e-05 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151129+0000", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2767000000000001e-05 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163899+0000", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012769951 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163907+0000", 2024-08-17T17:35:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.354e-06 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149776+0000", 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98586885099999999 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149784+0000", 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2039999999999997e-06 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370442+0000", 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.220658105 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370625+0000", 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018222600000000001 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370817+0000", 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019209699999999999 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.373489+0000", 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026719090000000001 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.375180+0000", 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016911890000000001 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380081+0000", 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049010909999999998 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380117+0000", 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6176000000000001e-05 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.383329+0000", 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003211864 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:151293 7.2 7:4fc50d10:::benchmark_data_smithi064_210574_object52484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:56.535631+0000", 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 150.73435090999999, 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18888440200000001, 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151293 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535631+0000", 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535631+0000", 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535633+0000", 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6759999999999999e-06 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535638+0000", 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2700000000000004e-06 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535651+0000", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2514000000000001e-05 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535658+0000", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1489999999999997e-06 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535801+0000", 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143714 2024-08-17T17:35:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.535952+0000", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150431 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.536054+0000", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101739 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.542143+0000", 2024-08-17T17:35:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060897499999999997 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.542243+0000", 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9557999999999998e-05 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.722749+0000", 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18050635500000001 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.724468+0000", 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001719199 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.724516+0000", 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7046999999999999e-05 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:151314 7.2 7:4e59723f:::benchmark_data_smithi064_210574_object52505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:56.568008+0000", 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 150.70197443999999, 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15666117700000001, 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151314 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568008+0000", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568008+0000", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568008+0000", 2024-08-17T17:35:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6199999999999997e-07 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568012+0000", 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.659e-06 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568022+0000", 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0438999999999999e-05 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568028+0000", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5899999999999998e-06 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568069+0000", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0472000000000001e-05 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568143+0000", 2024-08-17T17:35:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4305000000000001e-05 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.568206+0000", 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3188999999999994e-05 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.571588+0000", 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003382127 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.572850+0000", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012618569999999999 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.724592+0000", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15174237700000001 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.724638+0000", 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5389999999999997e-05 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:56.724669+0000", 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1010000000000003e-05 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193393 7.2 7:4557b385:::benchmark_data_smithi064_210574_object94584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.069169+0000", 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 84.200813496999999, 2024-08-17T17:35:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18388268899999999, 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193393 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069169+0000", 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069169+0000", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069170+0000", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.451e-06 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069175+0000", 2024-08-17T17:35:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5519999999999998e-06 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069187+0000", 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2228e-05 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.069195+0000", 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.077918+0000", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087234879999999997 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.078021+0000", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102484 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.078124+0000", 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103069 2024-08-17T17:35:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.082158+0000", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040344279999999996 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.100228+0000", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018069977000000001 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.248908+0000", 2024-08-17T17:35:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14868003699999999 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.253007+0000", 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040987819999999996 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.253051+0000", 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4320000000000003e-05 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193398 7.3 7:cf9ee9b3:::benchmark_data_smithi064_210574_object94589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.079325+0000", 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 84.190657379000001, 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17354751700000001, 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193398 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079325+0000", 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079325+0000", 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079326+0000", 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3519999999999999e-06 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079331+0000", 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5870000000000003e-06 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079343+0000", 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1929999999999999e-05 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079350+0000", 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0400000000000004e-06 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079594+0000", 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024429299999999997 2024-08-17T17:35:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079710+0000", 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116359 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.079821+0000", 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110868 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.085667+0000", 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058455790000000001 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088541+0000", 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028741169999999998 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.252756+0000", 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16421543399999999 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.252822+0000", 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5566000000000002e-05 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.252872+0000", 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0392000000000001e-05 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193405 7.a 7:5373d8f6:::benchmark_data_smithi064_210574_object94596:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.086933+0000", 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 84.183048826000004, 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.167460265, 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193405 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086933+0000", 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086933+0000", 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086935+0000", 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.55e-06 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086940+0000", 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9740000000000001e-06 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086951+0000", 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1636e-05 2024-08-17T17:35:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086958+0000", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9249999999999998e-06 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.087033+0000", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5187999999999999e-05 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.087140+0000", 2024-08-17T17:35:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010636700000000001 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.087244+0000", 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010428400000000001 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.100149+0000", 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012904491000000001 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.100272+0000", 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012378700000000001 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.254214+0000", 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15394128600000001 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.254349+0000", 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135351 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.254393+0000", 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4425999999999998e-05 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193407 7.a 7:56563ceb:::benchmark_data_smithi064_210574_object94598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.086979+0000", 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 84.183003365000005, 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18188249100000001, 2024-08-17T17:35:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193407 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086979+0000", 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086979+0000", 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086979+0000", 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2100000000000002e-07 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086981+0000", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2670000000000001e-06 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086987+0000", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0920000000000003e-06 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.086992+0000", 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6979999999999997e-06 2024-08-17T17:35:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.095524+0000", 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085314060000000001 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.095649+0000", 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012552599999999999 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:35:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.095763+0000", 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113924 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.100315+0000", 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045522729999999999 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.100764+0000", 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044839499999999998 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268776+0000", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16801187200000001 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268824+0000", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8078999999999998e-05 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268861+0000", 2024-08-17T17:35:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7537999999999999e-05 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193416 7.2 7:495b6c4f:::benchmark_data_smithi064_210574_object94607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.108794+0000", 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 84.161188136000007, 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.160188944, 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193416 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108794+0000", 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:35:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108794+0000", 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108795+0000", 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.483e-06 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108801+0000", 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8549999999999998e-06 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108815+0000", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.415e-05 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108823+0000", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1400000000000002e-06 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.108898+0000", 2024-08-17T17:35:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5951999999999994e-05 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.109043+0000", 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014468799999999999 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.109180+0000", 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013710400000000001 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.113430+0000", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042500369999999999 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.113441+0000", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1143e-05 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268906+0000", 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15546425799999999 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268947+0000", 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1390999999999997e-05 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.268983+0000", 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5743e-05 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:35:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:35:27.444 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:35:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:27.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:27.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:27.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.721 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:27.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:27.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:27.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:27.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:27.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:28.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:28.161 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 52755 52739 131.736 88.375 0.00734104 0.00758584 2024-08-17T17:35:28.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.285 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:28.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.429 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:28.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.530 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:28.559 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:28 smithi175 ceph-mon[101434]: pgmap v913: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-17T17:35:28.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:28.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:28.792 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:28.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:28.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:28 smithi064 ceph-mon[104516]: pgmap v913: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-17T17:35:28.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:28 smithi064 ceph-mon[109246]: pgmap v913: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-17T17:35:28.950 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:29.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.091 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.162 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 54267 54251 130.305 94.5 0.00974672 0.00766913 2024-08-17T17:35:29.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:29.224 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:29.326 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:29.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:29.457 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:29.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:29.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:29.526 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:35:29.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:29.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:29.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:29.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:29.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:29.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3664580311' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3664580311' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3664580311' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:29.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:30.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:30.120 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:30.161 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 55869 55853 129.188 100.125 0.00585749 0.00773482 2024-08-17T17:35:30.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:30.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:30.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:30.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:30.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:30.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[104516]: pgmap v914: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[104516]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[109246]: pgmap v914: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:30.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:30 smithi064 ceph-mon[109246]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:35:30.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:35:30.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:30.810 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:30 smithi175 ceph-mon[101434]: pgmap v914: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:35:30.810 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:30 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.810 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:30.810 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:30 smithi175 ceph-mon[101434]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:35:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:30.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:30.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:30.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:31.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:31.101 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:31.162 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 57371 57355 127.927 93.875 0.00619902 0.00781172 2024-08-17T17:35:31.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:31.266 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:31.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:31.407 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:31.493 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:31.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.545 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:31.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:31.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.687 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:31.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.827 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:31.965 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:32.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:32.080 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:32.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:32.162 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 58747 58731 126.482 86 0.00645738 0.00790081 2024-08-17T17:35:32.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:32.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:32.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:32.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:32.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:32.526 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:32.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:32.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:32.662 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:32.764 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:32.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:32.829 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:32 smithi175 ceph-mon[101434]: pgmap v916: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-08-17T17:35:32.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:32 smithi064 ceph-mon[104516]: pgmap v916: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-08-17T17:35:32.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:32 smithi064 ceph-mon[109246]: pgmap v916: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-08-17T17:35:32.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:32.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:33.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:33.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:33.162 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 60261 60245 125.42 94.625 0.0063921 0.007968 2024-08-17T17:35:33.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:33.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:33.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:33.351 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:33.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:33.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:33.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:33.619 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:33.691 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:61017 8.2 8:47f8a718:::benchmark_data_smithi064_242541_object61016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:33.685690+0000", 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0037319919999999999, 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037666039999999998, 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 61017 2024-08-17T17:35:33.692 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685690+0000", 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:35:33.693 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685690+0000", 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685688+0000", 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685749+0000", 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0470999999999998e-05 2024-08-17T17:35:33.694 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685762+0000", 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3657e-05 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685770+0000", 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7910000000000005e-06 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:35:33.695 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.685987+0000", 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000216925 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.686179+0000", 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019163700000000001 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.686305+0000", 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012661500000000001 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.696 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.688614+0000", 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023089230000000001 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.688903+0000", 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00028837099999999999 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.697 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:61021 8.4 e280/274 8:22bf3490:::benchmark_data_smithi064_242541_object61020:head v 280'3869, mlcod=280'3866)", 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:33.689323+0000", 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 9.8435000000000002e-05, 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000214043, 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689323+0000", 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:33.698 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689323+0000", 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689323+0000", 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:35:33.699 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689371+0000", 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8207000000000001e-05 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689385+0000", 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3603e-05 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689390+0000", 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1959999999999999e-06 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.700 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689421+0000", 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1510000000000002e-05 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:33.689439+0000", 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7801000000000001e-05 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:33.701 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:33.702 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:35:33.702 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 2 2024-08-17T17:35:33.702 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:35:33.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:33.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:33.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:33.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:34.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:34.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:35:34.148 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:34.163 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 61743 61727 124.357 92.625 0.0143157 0.00803596 2024-08-17T17:35:34.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:34.240 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:33] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.51.0" 2024-08-17T17:35:34.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:34.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:34.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:35:34.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:34.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:34.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:34 smithi175 ceph-mon[101434]: pgmap v917: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-17T17:35:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1546993461' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:34.716 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.740 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:34.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.828 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:34.829 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:34.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:34.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:34.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:34 smithi064 ceph-mon[104516]: pgmap v917: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-17T17:35:34.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1546993461' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:34.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:34 smithi064 ceph-mon[109246]: pgmap v917: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-17T17:35:34.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1546993461' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:34.831 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:35:34.831 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:35:34.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:34.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:34.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:35.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:35.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:35.164 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:35.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 63046 63030 123.016 81.4375 0.00381478 0.00812352 2024-08-17T17:35:35.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:35.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:35.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:35.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:35.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:35.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:35.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:35.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.844 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:35.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.163 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 64197 64181 121.469 71.9375 0.00807042 0.00822742 2024-08-17T17:35:36.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:36.264 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:36.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:36.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:36.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:36.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.529 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:36.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:36.563 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:36 smithi175 ceph-mon[101434]: pgmap v918: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-08-17T17:35:36.630 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:36.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:36.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:36.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:36.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:36.802 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:36.802 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:36.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:36.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:36 smithi064 ceph-mon[104516]: pgmap v918: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-08-17T17:35:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:36 smithi064 ceph-mon[109246]: pgmap v918: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-08-17T17:35:37.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:37.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:37.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:37.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:37.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:37.163 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 65007 64991 119.387 50.625 0.0132188 0.00837131 2024-08-17T17:35:37.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:37.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:37.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:37.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:37.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:37.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:37.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:37.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:37.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:37.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:38.062 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.097 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:38.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:38.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 66223 66207 118.148 76 0.0120046 0.00845895 2024-08-17T17:35:38.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:38.232 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:38.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:38.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:38.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:38.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:38.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:38.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:38 smithi175 ceph-mon[101434]: pgmap v919: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:35:38.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:35:38.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:38.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:38.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:38.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:38.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:38 smithi064 ceph-mon[104516]: pgmap v919: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:35:38.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:38 smithi064 ceph-mon[109246]: pgmap v919: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:35:39.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:39.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:39.144 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:39.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:39.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 67553 67537 117.175 83.125 0.00328841 0.00851572 2024-08-17T17:35:39.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:39.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:39.330 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:39.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:39.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:39.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:39.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:39.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:39.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:39.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:39.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:39.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:39.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:40.005 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:40.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:40.146 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:40.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 68787 68771 116.093 77.125 0.00359929 0.00860814 2024-08-17T17:35:40.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:40.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:40.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:40.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:40.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:40.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:40.619 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:40.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.641 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:40.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:40.658 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:40 smithi175 ceph-mon[101434]: pgmap v920: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:35:40.658 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:40.658 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:40.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:40.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:40.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:40.910 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[104516]: pgmap v920: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:35:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[109246]: pgmap v920: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:35:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:35:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:40.943 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:40.943 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:41.012 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:41.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:41.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:41.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 69771 69755 114.655 61.5 0.00647263 0.00871655 2024-08-17T17:35:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:41.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:41.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:41.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:41.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:41.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:41.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:41.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:41.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:41.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:41.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:41.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:41.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:41.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:41.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:41.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:42.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:42.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:42.164 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 15 70679 70664 113.173 56.8125 0.00398889 0.00882927 2024-08-17T17:35:42.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:42.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:42.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:42.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:42.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:42.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:42.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:42.633 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:35:42.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:42.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:42.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:42.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:42.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:42 smithi175 ceph-mon[101434]: pgmap v921: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:35:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:42 smithi064 ceph-mon[104516]: pgmap v921: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:35:42.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:42 smithi064 ceph-mon[109246]: pgmap v921: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:35:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:42.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:42.968 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:43.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:43.089 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:43.165 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:35:43.165064+0000 min lat: 0.00104069 max lat: 0.388376 avg lat: 0.0088973 2024-08-17T17:35:43.165 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:43.165 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 71955 71939 112.336 79.6875 0.0147901 0.0088973 2024-08-17T17:35:43.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:43.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:43.283 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:43.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:43.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:43.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:43.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:43.523 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:43.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:43.657 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:43.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:43.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:43.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:43.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:43.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:43.935 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:43.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:44.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:44.085 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:44.165 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 73256 73240 111.58 81.3125 0.00653489 0.00895729 2024-08-17T17:35:44.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:44.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:44.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:44.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:43] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.51.0" 2024-08-17T17:35:44.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:44.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:44.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:44.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:44.709 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:44 smithi175 ceph-mon[101434]: pgmap v922: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:35:44.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:44.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:44.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:44 smithi064 ceph-mon[104516]: pgmap v922: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:35:44.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:44 smithi064 ceph-mon[109246]: pgmap v922: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:35:44.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:44.837 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:44.838 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:44.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:44.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:45.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:45.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:45.166 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 74535 74519 110.823 79.9375 0.00672702 0.00901777 2024-08-17T17:35:45.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:45.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.434 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:45.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:45.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:45.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-08-17T17:35:45.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:35:45.566 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:45.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:45.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:45.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:45.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:45.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:45.870 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:45.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:45.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:46.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:46.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:46.157 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:46.166 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 75756 75740 110.021 76.3125 0.0086369 0.00908339 2024-08-17T17:35:46.214 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:46.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:46.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:46.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:46.440 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:46.542 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:46.556 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:35:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:46.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:46.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[104516]: pgmap v923: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-17T17:35:46.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4202691731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[109246]: pgmap v923: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-17T17:35:46.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4202691731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:46.713 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:46.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:46.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:46 smithi175 ceph-mon[101434]: pgmap v923: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-17T17:35:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4202691731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:46.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:46.991 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:47.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.166 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 76970 76954 109.245 75.875 0.00614262 0.00914856 2024-08-17T17:35:47.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:47.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:47.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:47.551 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:47.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:47.684 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:47.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[104516]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:35:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:47.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:47 smithi064 ceph-mon[109246]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:35:47.713 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:47 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:47.713 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:47.713 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:47 smithi175 ceph-mon[101434]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:35:47.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:47.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:47.863 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:47.863 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:47.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:47.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.166 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 78165 78149 108.477 74.6875 0.0280886 0.00921252 2024-08-17T17:35:48.200 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:48.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:48.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:48.506 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:48.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:35:48.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:48.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:48.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:48.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:48 smithi175 ceph-mon[101434]: pgmap v925: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:35:48.928 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:48.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:48.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:48 smithi064 ceph-mon[104516]: pgmap v925: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:35:48.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:48 smithi064 ceph-mon[109246]: pgmap v925: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:35:49.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:49.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:49.167 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 79456 79440 107.873 80.6875 0.0151863 0.00926494 2024-08-17T17:35:49.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:49.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:49.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:49.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:49.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:49.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:49.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:49.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:49.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:49.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:49.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:49.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:49.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:49.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:50.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:50.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:50.160 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:50.167 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 80717 80701 107.255 78.8125 0.012247 0.00931864 2024-08-17T17:35:50.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:50.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:50.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:50.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:50.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:50.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:50.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:50 smithi064 ceph-mon[104516]: pgmap v926: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:35:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:50 smithi064 ceph-mon[109246]: pgmap v926: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:35:50.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:50.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:50.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:50.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:50.871 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:50 smithi175 ceph-mon[101434]: pgmap v926: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:35:50.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.996 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:50.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:51.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:51.134 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:51.167 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 81854 81838 106.501 71.0625 0.0137267 0.00938495 2024-08-17T17:35:51.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.435 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.509 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:51.509 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:51.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:51.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:51.616 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:35:51.616 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:51.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:35:51.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:51 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3645004228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:51 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3645004228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:51.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.702 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:35:51.703 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.738 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:35:51.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:35:51.771 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:51.815 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:35:51.815 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:51.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:51.850 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:35:51.851 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.886 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:35:51.886 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:51.889 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:51 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3645004228' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:51.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:35:51.952 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:51.979 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:51.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:52.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:52.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 82828 82812 105.563 60.875 0.00541324 0.00946811 2024-08-17T17:35:52.211 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:52.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:35:52.246 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:52.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:52.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:52.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:52.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:52.567 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:52.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.659 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.761 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:52.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:52 smithi175 ceph-mon[101434]: pgmap v927: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:35:52.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:52 smithi064 ceph-mon[104516]: pgmap v927: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:35:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:52 smithi064 ceph-mon[109246]: pgmap v927: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:35:53.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:53.068 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:53.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.172 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 83972 83956 104.882 71.5 0.00439106 0.00952484 2024-08-17T17:35:53.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.194 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:53.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:53.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:53.610 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:53.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:53.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:53.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:53.893 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:53.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:54.038 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:54.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:54.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 85091 85075 104.197 69.9375 0.00500608 0.00959153 2024-08-17T17:35:54.176 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:54.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:54.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:54.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:54.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:35:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:35:53] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.51.0" 2024-08-17T17:35:54.438 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:54.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:54.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:54.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:54.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:54.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:54.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:54.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:54 smithi175 ceph-mon[101434]: pgmap v928: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:35:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:54 smithi064 ceph-mon[104516]: pgmap v928: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:35:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:54 smithi064 ceph-mon[109246]: pgmap v928: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:35:54.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:54.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:55.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:55.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 85991 85975 103.276 56.25 0.0156608 0.00967748 2024-08-17T17:35:55.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:55.254 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:55.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.394 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:55.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-08-17T17:35:55.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:55.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:55.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:55.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:55.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:55.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:55.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:55.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.984 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:55.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:56.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:56.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:56.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 87003 86987 102.521 63.25 0.00516724 0.00974899 2024-08-17T17:35:56.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:87059 8.4 8:204f139b:::benchmark_data_smithi064_242541_object87058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:56.218857+0000", 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0093051560000000002, 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093392660000000006, 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:35:56.230 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 87059 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218857+0000", 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.231 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218857+0000", 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:35:56.232 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218855+0000", 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218912+0000", 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6719999999999999e-05 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218929+0000", 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6949999999999999e-05 2024-08-17T17:35:56.233 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.218937+0000", 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9990000000000008e-06 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.219008+0000", 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.093e-05 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.234 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.219137+0000", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012952800000000001 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.219301+0000", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016391299999999999 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.225824+0000", 2024-08-17T17:35:56.235 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0065228930000000001 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.225850+0000", 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6194999999999999e-05 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:87061 8.4 8:2f3212c7:::benchmark_data_smithi064_242541_object87060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:35:56.236 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:56.221339+0000", 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0068230879999999997, 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0069446480000000003, 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 87061 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:35:56.237 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221339+0000", 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221339+0000", 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.238 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221337+0000", 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221373+0000", 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6319000000000002e-05 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:35:56.239 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221390+0000", 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6668999999999999e-05 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221397+0000", 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5610000000000001e-06 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.221997+0000", 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00060000299999999999 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.240 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.222129+0000", 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000131803 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.222292+0000", 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016234499999999999 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:56.241 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.225881+0000", 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035890470000000002 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.225951+0000", 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0549000000000005e-05 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:87069 8.2 e281/274 8:4f6662c3:::benchmark_data_smithi064_242541_object87068:head v 281'5454, mlcod=281'5452)", 2024-08-17T17:35:56.242 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:56.227749+0000", 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.00041283500000000001, 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00060924099999999995, 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227749+0000", 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:35:56.243 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227749+0000", 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227748+0000", 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227769+0000", 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1067e-05 2024-08-17T17:35:56.244 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227781+0000", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2294e-05 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227788+0000", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8199999999999999e-06 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227886+0000", 2024-08-17T17:35:56.245 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7893000000000006e-05 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:56.227916+0000", 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0114999999999999e-05 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 3 2024-08-17T17:35:56.246 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:35:56.265 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:56.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:56.400 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:56.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:56.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:56.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:56.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:56.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:56.714 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:56.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:56.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:56.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:56 smithi064 ceph-mon[104516]: pgmap v929: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:35:56.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:56 smithi064 ceph-mon[109246]: pgmap v929: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:35:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:56 smithi175 ceph-mon[101434]: pgmap v929: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:35:56.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:56.995 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:57.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:35:57.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:57.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:57.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 88096 88080 101.887 68.3125 0.00849209 0.00980895 2024-08-17T17:35:57.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:57.289 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:57.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:57.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:57.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:57.475 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:57.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:57.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:57.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:57.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:57.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:57.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:57.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:57.930 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:57.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:58.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:58.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:58.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 89209 89193 101.3 69.5625 0.0137498 0.00986635 2024-08-17T17:35:58.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:58.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:58.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:58.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.481 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:58.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:58 smithi064 ceph-mon[104516]: pgmap v930: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T17:35:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:58 smithi064 ceph-mon[109246]: pgmap v930: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T17:35:58.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:58 smithi175 ceph-mon[101434]: pgmap v930: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T17:35:58.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:59.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:59.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:59.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:59.170 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 90220 90204 100.62 63.1875 0.00899625 0.0099331 2024-08-17T17:35:59.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:59.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:59.329 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:59.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:59.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:59.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:59.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:59.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:59.653 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:35:59.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:59.722 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:35:59.722 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:35:59.722 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:35:59.722 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:35:59.722 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:35:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.746 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:35:59.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:59.847 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:59.868 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:59.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2031571926' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:35:59 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2031571926' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:35:59 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2031571926' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:35:59 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:59.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:59.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:35:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:00.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 91066 91050 99.7822 52.875 0.0108136 0.0100167 2024-08-17T17:36:00.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.257 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:00.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:00.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:00.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:00.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:00.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:00.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:00.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:00.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:00.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:00 smithi175 ceph-mon[101434]: pgmap v931: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:00 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:00 smithi175 ceph-mon[101434]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:36:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[104516]: pgmap v931: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[104516]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:36:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[109246]: pgmap v931: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:00 smithi064 ceph-mon[109246]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:36:00.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.994 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:00.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:01.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.133 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:01.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:01.165 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:36:01.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 15 92166 92151 99.2483 68.8125 0.0121353 0.0100704 2024-08-17T17:36:01.234 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:01.253 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:01.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:01.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:01.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:01.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:01.512 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:01.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:01.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:01.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:01.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:01.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:01.856 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:01.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:01.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:01.999 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:02.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:02.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:02.172 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 93292 93276 98.758 70.3125 0.0146426 0.0101206 2024-08-17T17:36:02.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:02.281 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:02.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:02.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:02.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:02.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:02.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:02.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:02.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:02.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:02.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:02.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:02.852 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:02 smithi175 ceph-mon[101434]: pgmap v933: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:36:02.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-08-17T17:36:02.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:02 smithi064 ceph-mon[104516]: pgmap v933: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:36:02.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:02 smithi064 ceph-mon[109246]: pgmap v933: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:36:02.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:02.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:03.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:36:03.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:03.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:03.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:36:03.171420+0000 min lat: 0.00104069 max lat: 0.388376 avg lat: 0.010172 2024-08-17T17:36:03.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:36:03.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 94389 94373 98.2548 68.5625 0.00871605 0.010172 2024-08-17T17:36:03.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:03.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:03.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:03.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.456 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:113447 7.0 7:0238920a:::benchmark_data_smithi064_210574_object14638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:01.330786+0000", 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 242.12193259, 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23208737099999999, 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 113447 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.457 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330786+0000", 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330786+0000", 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330784+0000", 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330794+0000", 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6690000000000001e-06 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330813+0000", 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9011000000000001e-05 2024-08-17T17:36:03.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330820+0000", 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.357e-06 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330866+0000", 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5853999999999998e-05 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.330961+0000", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5057999999999997e-05 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.331034+0000", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2435999999999996e-05 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.333226+0000", 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021917629999999998 2024-08-17T17:36:03.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.334041+0000", 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00081586600000000005 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.562776+0000", 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.228734613 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.562833+0000", 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6839999999999998e-05 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:01.562874+0000", 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0769999999999998e-05 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:151292 7.0 7:02252cbb:::benchmark_data_smithi064_210574_object52483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:56.532092+0000", 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 186.92062722899999, 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18737225800000001, 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.464 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151292 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532092+0000", 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.465 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532092+0000", 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532090+0000", 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532096+0000", 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0739999999999998e-06 2024-08-17T17:36:03.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532105+0000", 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4039999999999995e-06 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532112+0000", 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0840000000000003e-06 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532144+0000", 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1745999999999997e-05 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532231+0000", 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7627999999999995e-05 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.532306+0000", 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4412000000000005e-05 2024-08-17T17:36:03.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.536027+0000", 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037213419999999999 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.547395+0000", 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011367776 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.719359+0000", 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.171964065 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.719417+0000", 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8369999999999998e-05 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.719464+0000", 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6666999999999999e-05 2024-08-17T17:36:03.470 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:151299 7.0 7:00bae1e5:::benchmark_data_smithi064_210574_object52490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:56.540238+0000", 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 186.91248132300001, 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18390677999999999, 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:03.471 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151299 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540238+0000", 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540238+0000", 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540236+0000", 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540243+0000", 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.781e-06 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540254+0000", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1386000000000001e-05 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540260+0000", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6550000000000001e-06 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540302+0000", 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1702000000000002e-05 2024-08-17T17:36:03.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540437+0000", 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013472199999999999 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.540568+0000", 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013191099999999999 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.543919+0000", 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033509439999999998 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.547436+0000", 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035163460000000001 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724053+0000", 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.176616788 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724102+0000", 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9715000000000001e-05 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724144+0000", 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2043999999999999e-05 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:151311 7.0 7:0508aa2f:::benchmark_data_smithi064_210574_object52502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:56.566578+0000", 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 186.88614036000001, 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.15768543600000001, 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151311 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.478 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566578+0000", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566578+0000", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566577+0000", 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566587+0000", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0008e-05 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566598+0000", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1253000000000001e-05 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566604+0000", 2024-08-17T17:36:03.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9780000000000002e-06 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566658+0000", 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3408000000000001e-05 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566816+0000", 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015848699999999999 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:03.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.566959+0000", 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014243599999999999 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.570795+0000", 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038360540000000002 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.571326+0000", 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00053085999999999997 2024-08-17T17:36:03.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724187+0000", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.152861782 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724231+0000", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3798999999999999e-05 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:56.724264+0000", 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2824999999999998e-05 2024-08-17T17:36:03.483 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:194445 7.0 7:04958d83:::benchmark_data_smithi064_210574_object95636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:34:04.797175+0000", 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 118.655543943, 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.098304381999999996, 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 194445 2024-08-17T17:36:03.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797175+0000", 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797175+0000", 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797173+0000", 2024-08-17T17:36:03.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:03.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797180+0000", 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1489999999999997e-06 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797196+0000", 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5540999999999999e-05 2024-08-17T17:36:03.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797203+0000", 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4200000000000001e-06 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797249+0000", 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5510000000000003e-05 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797357+0000", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107817 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.797490+0000", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013307099999999999 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.802031+0000", 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004541542 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.803896+0000", 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0018643209999999999 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.895408+0000", 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.091512283 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.490 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.895440+0000", 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2379000000000003e-05 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:34:04.895479+0000", 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9020999999999998e-05 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57275 8.d 8:beccb93b:::benchmark_data_smithi064_242541_object57274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:36:03.491 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.026696+0000", 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 32.426022750999998, 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.116610535, 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57275 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.492 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026696+0000", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026696+0000", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026696+0000", 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026749+0000", 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3888999999999999e-05 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026760+0000", 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0484e-05 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.494 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.026765+0000", 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4750000000000001e-06 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.029232+0000", 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0024661420000000002 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.029349+0000", 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000117674 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.029485+0000", 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013538600000000001 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.031692+0000", 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022070789999999998 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035464+0000", 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003772297 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143212+0000", 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.107747994 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143257+0000", 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5114999999999997e-05 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143307+0000", 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9589999999999998e-05 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57282 8.8 8:1965d1be:::benchmark_data_smithi064_242541_object57281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.030053+0000", 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 32.422665750999997, 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.112026581, 2024-08-17T17:36:03.498 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57282 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030053+0000", 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030053+0000", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030052+0000", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030371+0000", 2024-08-17T17:36:03.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00031871200000000002 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030382+0000", 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0701e-05 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030388+0000", 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4010000000000002e-06 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.501 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030451+0000", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2800999999999997e-05 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030560+0000", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010935899999999999 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.030677+0000", 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011683099999999999 2024-08-17T17:36:03.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.033971+0000", 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032940299999999999 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.034898+0000", 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00092696200000000003 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142002+0000", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.107104268 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142045+0000", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2706000000000001e-05 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142080+0000", 2024-08-17T17:36:03.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4567999999999998e-05 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57283 8.d 8:bf8fc1f5:::benchmark_data_smithi064_242541_object57282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.032508+0000", 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 32.420211080999998, 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.110940944, 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.505 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57283 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032508+0000", 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032508+0000", 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032507+0000", 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032537+0000", 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0849000000000002e-05 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032550+0000", 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2079000000000001e-05 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.508 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032556+0000", 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6590000000000001e-06 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032592+0000", 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5586000000000001e-05 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032703+0000", 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011145300000000001 2024-08-17T17:36:03.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.032808+0000", 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104941 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.034709+0000", 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0019008320000000001 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.042614+0000", 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079047449999999995 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143365+0000", 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.100751128 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143413+0000", 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8374000000000001e-05 2024-08-17T17:36:03.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.143449+0000", 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5479999999999999e-05 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57290 8.d 8:b5134085:::benchmark_data_smithi064_242541_object57289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.035581+0000", 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 32.417137613000001, 2024-08-17T17:36:03.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.10992558299999999, 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57290 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035581+0000", 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035581+0000", 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035580+0000", 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035639+0000", 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8346e-05 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035648+0000", 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5550000000000008e-06 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035653+0000", 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8389999999999997e-06 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035676+0000", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3125999999999998e-05 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035759+0000", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2909000000000002e-05 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.035858+0000", 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9222000000000006e-05 2024-08-17T17:36:03.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.037601+0000", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001742988 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.042642+0000", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0050407389999999998 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.145435+0000", 2024-08-17T17:36:03.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.10279276 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.145473+0000", 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.862e-05 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.145507+0000", 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3448999999999998e-05 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.518 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57298 8.8 8:1fbe187d:::benchmark_data_smithi064_242541_object57297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.044474+0000", 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 32.408245084000001, 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.098461818000000007, 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.519 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57298 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044474+0000", 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044474+0000", 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044473+0000", 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044518+0000", 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5364999999999997e-05 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044529+0000", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0369e-05 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044533+0000", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5950000000000001e-06 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044585+0000", 2024-08-17T17:36:03.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1662000000000001e-05 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044711+0000", 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012662700000000001 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.044866+0000", 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015485799999999999 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.046474+0000", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001607491 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.050805+0000", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043310520000000002 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142859+0000", 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.092053731 2024-08-17T17:36:03.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142903+0000", 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4400000000000002e-05 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.142936+0000", 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2638999999999997e-05 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85769 8.4 8:2e046255:::benchmark_data_smithi064_242541_object85768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.824530+0000", 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.6281892060000001, 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.20245640300000001, 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85769 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824530+0000", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824530+0000", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824528+0000", 2024-08-17T17:36:03.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824579+0000", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0745999999999998e-05 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824594+0000", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5051e-05 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824601+0000", 2024-08-17T17:36:03.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824648+0000", 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6631999999999997e-05 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824781+0000", 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013320000000000001 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.825071+0000", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00029025600000000002 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.830319+0000", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.005247959 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.833253+0000", 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0029338150000000002 2024-08-17T17:36:03.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.026892+0000", 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.193638902 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.026947+0000", 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5041999999999999e-05 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.026986+0000", 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9286999999999998e-05 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85770 8.4 8:251677ab:::benchmark_data_smithi064_242541_object85769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.824863+0000", 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.6278557189999994, 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.21405238500000001, 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.532 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85770 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824863+0000", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824863+0000", 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824862+0000", 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824904+0000", 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1106000000000003e-05 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824916+0000", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2131e-05 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.824923+0000", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0299999999999996e-06 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.827714+0000", 2024-08-17T17:36:03.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0027914929999999999 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.827796+0000", 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1996999999999994e-05 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.827905+0000", 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000108699 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.830445+0000", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025402210000000001 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.834891+0000", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004446017 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.038823+0000", 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.20393190899999999 2024-08-17T17:36:03.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.038877+0000", 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3782000000000001e-05 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.038916+0000", 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8668000000000003e-05 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85781 8.4 8:220438b9:::benchmark_data_smithi064_242541_object85780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.835213+0000", 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.6175062239999995, 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.20433270100000001, 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85781 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.539 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835213+0000", 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835213+0000", 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835212+0000", 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835313+0000", 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101297 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835323+0000", 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4010000000000003e-06 2024-08-17T17:36:03.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835328+0000", 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5450000000000003e-06 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835361+0000", 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3241999999999998e-05 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835446+0000", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4240999999999995e-05 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.835547+0000", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101566 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.837751+0000", 2024-08-17T17:36:03.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022040100000000002 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.841495+0000", 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003743222 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039464+0000", 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.197969006 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039512+0000", 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.867e-05 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039545+0000", 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3141e-05 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.545 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85789 8.4 8:20dff97a:::benchmark_data_smithi064_242541_object85788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.842325+0000", 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.6103935610000004, 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19733252600000001, 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85789 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.546 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842325+0000", 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842325+0000", 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842324+0000", 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842385+0000", 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0955999999999997e-05 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842395+0000", 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5920000000000007e-06 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842399+0000", 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9999999999999998e-06 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842426+0000", 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7155999999999999e-05 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842532+0000", 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105684 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.842628+0000", 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6496000000000006e-05 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.844981+0000", 2024-08-17T17:36:03.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023526889999999998 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.846313+0000", 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0013320109999999999 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039582+0000", 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19326955500000001 2024-08-17T17:36:03.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039626+0000", 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3127999999999998e-05 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.039658+0000", 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2299999999999999e-05 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85794 8.4 8:2992969d:::benchmark_data_smithi064_242541_object85793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.849226+0000", 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.6034926299999999, 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19376468599999999, 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85794 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.553 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849226+0000", 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849226+0000", 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849225+0000", 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849289+0000", 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3250000000000006e-05 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849301+0000", 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2289e-05 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849306+0000", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2020000000000003e-06 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849348+0000", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1996999999999998e-05 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849473+0000", 2024-08-17T17:36:03.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000125182 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.849616+0000", 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142418 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.851836+0000", 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022203800000000001 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.853301+0000", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014645529999999999 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.042908+0000", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18960773 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.042954+0000", 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5646999999999999e-05 2024-08-17T17:36:03.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.042991+0000", 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6767000000000002e-05 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85807 8.4 8:288b4f97:::benchmark_data_smithi064_242541_object85806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.866834+0000", 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 8.5858844179999991, 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.176281992, 2024-08-17T17:36:03.559 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85807 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866834+0000", 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866834+0000", 2024-08-17T17:36:03.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:03.562 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:03.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866834+0000", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866887+0000", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3427999999999997e-05 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866898+0000", 2024-08-17T17:36:03.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0761000000000001e-05 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866902+0000", 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3619999999999999e-06 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.866936+0000", 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3637000000000003e-05 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.867035+0000", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9569999999999997e-05 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.867127+0000", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1922000000000005e-05 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.869435+0000", 2024-08-17T17:36:03.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023079369999999999 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.870026+0000", 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00059058500000000002 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.043050+0000", 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.17302413899999999 2024-08-17T17:36:03.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.043088+0000", 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8059000000000003e-05 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.043116+0000", 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8450999999999999e-05 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90937 8.9 8:9bed6930:::benchmark_data_smithi064_242541_object90936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.807527+0000", 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6451915779999999, 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29212719799999998, 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90937 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807527+0000", 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807527+0000", 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807527+0000", 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807578+0000", 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0686999999999998e-05 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807585+0000", 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.52e-06 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.807590+0000", 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1950000000000002e-06 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.815245+0000", 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0076548329999999998 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.815352+0000", 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000106858 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.815489+0000", 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000137348 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818561+0000", 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030713200000000002 2024-08-17T17:36:03.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.826681+0000", 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0081207459999999999 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099423+0000", 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27274134700000002 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099606+0000", 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000182826 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099654+0000", 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8940000000000002e-05 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90938 8.9 8:992b444a:::benchmark_data_smithi064_242541_object90937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:03.574 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.808374+0000", 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6443447500000001, 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29139544299999998, 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90938 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808374+0000", 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808374+0000", 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808373+0000", 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808407+0000", 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3859000000000002e-05 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808418+0000", 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0902e-05 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.808424+0000", 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6019999999999996e-06 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818584+0000", 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010159919999999999 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818687+0000", 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010241799999999999 2024-08-17T17:36:03.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818800+0000", 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000113639 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.826545+0000", 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0077441610000000003 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.826736+0000", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000191762 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099696+0000", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27295970800000002 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099738+0000", 2024-08-17T17:36:03.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2286999999999997e-05 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099770+0000", 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1238000000000001e-05 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90947 8.9 8:955f1dde:::benchmark_data_smithi064_242541_object90946:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.817278+0000", 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6354413060000002, 2024-08-17T17:36:03.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28264444300000002, 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90947 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817278+0000", 2024-08-17T17:36:03.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817278+0000", 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817276+0000", 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817315+0000", 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9542999999999997e-05 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817327+0000", 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1732e-05 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.817332+0000", 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3550000000000004e-06 2024-08-17T17:36:03.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.821545+0000", 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042129569999999998 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.821677+0000", 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013160400000000001 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:03.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.821837+0000", 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015998600000000001 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.826549+0000", 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0047118730000000001 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.829054+0000", 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025047260000000001 2024-08-17T17:36:03.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099804+0000", 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27074989599999999 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099889+0000", 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5063999999999997e-05 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099922+0000", 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3395999999999997e-05 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90951 8.9 8:9e3a43fa:::benchmark_data_smithi064_242541_object90950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.818684+0000", 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6340343819999998, 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28138680999999999, 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:03.588 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90951 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818684+0000", 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:36:03.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818684+0000", 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818684+0000", 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818797+0000", 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011355 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818811+0000", 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4042e-05 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.818819+0000", 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2649999999999999e-06 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:36:03.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.824278+0000", 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0054595859999999998 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.824362+0000", 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4160999999999996e-05 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.824443+0000", 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0678000000000002e-05 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.826692+0000", 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022488019999999998 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.829195+0000", 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025026269999999999 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:03.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.099964+0000", 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27076997200000003 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.100035+0000", 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0031999999999996e-05 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.100071+0000", 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6755999999999998e-05 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.594 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.595 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.595 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:36:03.595 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:36:03.595 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:36:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.649 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:03.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:03.742 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:03.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:03.949 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:04.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:04.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:04.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 95357 95341 97.6361 60.5 0.00618198 0.0102374 2024-08-17T17:36:04.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:03] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.51.0" 2024-08-17T17:36:04.207 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.309 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:04.345 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.447 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:04.479 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:04.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:04.725 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:04.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:04.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:04.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:36:04.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:04 smithi175 ceph-mon[101434]: pgmap v934: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:36:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:04 smithi064 ceph-mon[104516]: pgmap v934: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:36:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:04 smithi064 ceph-mon[109246]: pgmap v934: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:36:05.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:05.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:05.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:05.239 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 96468 96452 97.1813 69.4375 0.0105942 0.0102845 2024-08-17T17:36:05.251 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.288 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:05.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:05.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:05.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:05.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:05.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:05.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:05.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:05.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:05.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2160302522' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:05.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2160302522' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:05.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:05.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:05.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.990 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:05.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:06.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:06.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.147 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:06.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2160302522' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:06.172 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 97558 97542 96.7184 68.125 0.0292558 0.0103339 2024-08-17T17:36:06.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:06.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:06.376 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:36:06.376 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:06.377 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:06.378 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:06.378 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:06.378 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:36:06.378 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:06.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.554 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:06.700 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:06 smithi175 ceph-mon[101434]: pgmap v935: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:06.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:06.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.846 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:06.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:06 smithi064 ceph-mon[104516]: pgmap v935: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:06.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:06 smithi064 ceph-mon[109246]: pgmap v935: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:06.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:06.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:07.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:07.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:07.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:07.150 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:07.150 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:07.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 98686 98670 96.3088 70.5 0.0124273 0.0103782 2024-08-17T17:36:07.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:07.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.269 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:07.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:07.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:07.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:07.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:07.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:07.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:07.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:07.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:07.817 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:07.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:36:07.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:07.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:07.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:08.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:08.091 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:08.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:08.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 99716 99700 95.8176 64.375 0.0139777 0.0104309 2024-08-17T17:36:08.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:08.205 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:08.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:08.332 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:08.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:08.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:08.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:08.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:08.633 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:08.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:08.783 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:08.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:08.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:08 smithi175 ceph-mon[101434]: pgmap v936: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:08.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:08.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:08 smithi064 ceph-mon[104516]: pgmap v936: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:08.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:08 smithi064 ceph-mon[109246]: pgmap v936: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:09.014 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:09.042 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:09.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:09.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 100644 100628 95.2444 58 0.0149271 0.0104941 2024-08-17T17:36:09.185 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:09.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:09.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:09.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:09.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:09.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:09.464 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:09.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:09.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:09.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.601 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:09.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:09.690 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:36:09.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:09.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:09.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:09.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:09.871 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:09.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-08-17T17:36:09.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:10.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:10.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:10.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:10.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 101578 101562 94.6942 58.375 0.0276527 0.0105547 2024-08-17T17:36:10.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:10.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:10.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:10.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:10.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:10.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:10.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:10.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:10.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: pgmap v937: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: pgmap v937: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:10.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:36:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:36:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:10.706 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:10.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:10.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:10.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:10.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:10.971 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:11.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: pgmap v937: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:36:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2010487519' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:11.118 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:36:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:11.119 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:11.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 102449 102433 94.1024 54.4375 0.00648571 0.0106209 2024-08-17T17:36:11.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:11.242 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:11.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:11.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:11.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:11.419 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:11.419 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:11.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:11.499 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:11.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:11.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:11.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:11.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:11.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:11.778 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:11.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:11.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:12.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:12.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:12.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:12.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 103511 103495 93.7006 66.375 0.0246486 0.0106671 2024-08-17T17:36:12.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.193 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:12.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:36:12.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:12.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:12.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:12.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:12.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:12.601 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:12.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:36:12.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:12.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:12.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:12.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:12 smithi064 ceph-mon[104516]: pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:12.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:12 smithi064 ceph-mon[109246]: pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:12.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.025 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:13.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:13.031 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:12 smithi175 ceph-mon[101434]: pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:36:13.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:13.164 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:13.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 104513 104497 93.2568 62.625 0.0145976 0.0107176 2024-08-17T17:36:13.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:13.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:13.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.464 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:13.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:13.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:13.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:13.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:13.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:13.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:13.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:13.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:13.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:13.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:14.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:14.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:14.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:14.173 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 105392 105376 92.7171 54.9375 0.00449808 0.0107615 2024-08-17T17:36:14.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:13] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-08-17T17:36:14.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:14.294 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:14.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:14.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:14.570 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:14.570 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:14.580 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:14.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:14.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:14.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:14.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:14 smithi064 ceph-mon[104516]: pgmap v940: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:36:14.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3022080482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:14 smithi064 ceph-mon[109246]: pgmap v940: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:36:14.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3022080482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:14.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:14.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:14.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:14 smithi175 ceph-mon[101434]: pgmap v940: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:36:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3022080482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:14.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:15.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:15.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:15.174 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 106346 106330 92.2576 59.625 0.0158313 0.010834 2024-08-17T17:36:15.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:15.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:15.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:36:15.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:15.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:15.527 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:15.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:15.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:15.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:15.804 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:15.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:15.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:15.935 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:15.962 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:36:15.962 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:36:15.962 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:15.963 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:15.964 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:15.964 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:15.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.966 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:15.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:16.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.094 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:16.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:16.174 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 107322 107306 91.8295 61 0.0109422 0.0108845 2024-08-17T17:36:16.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:16.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:16.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:16.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:16.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:16.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:16.608 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:16.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:16.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:16.765 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:16.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:16.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:16.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:17.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:17.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:17.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:16 smithi175 ceph-mon[101434]: pgmap v941: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:17.174 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 108235 108219 91.3598 57.0625 0.0652064 0.0109386 2024-08-17T17:36:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:16 smithi064 ceph-mon[104516]: pgmap v941: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:17.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:16 smithi064 ceph-mon[109246]: pgmap v941: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:36:17.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:17.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:17.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:17.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:17.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:17.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:17.541 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:17.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:17.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:17.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:17.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:17.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:17.970 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:18.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:18.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:18.174 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 109234 109218 90.9742 62.4375 0.0150938 0.0109868 2024-08-17T17:36:18.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:18.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:18.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:18.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:18.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:18.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:18.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:18.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:18.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.919 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:19.021 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:19.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:19.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:18 smithi064 ceph-mon[104516]: pgmap v942: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-08-17T17:36:19.129 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:18 smithi064 ceph-mon[109246]: pgmap v942: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-08-17T17:36:19.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:19.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:18 smithi175 ceph-mon[101434]: pgmap v942: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-08-17T17:36:19.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 110096 110080 90.486 53.875 0.0259018 0.0110451 2024-08-17T17:36:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:19.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:19.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:19.318 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:19.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:19.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:19.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:19.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:19.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:19.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:19.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:19.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:19.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:20.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:20.105 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:20.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:20.174 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 111063 111047 90.0958 60.4375 0.00653475 0.0110941 2024-08-17T17:36:20.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:20.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:20.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:20.415 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:20.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:20.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:20.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:20.833 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:20.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-08-17T17:36:20.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:20.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:20.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:20 smithi064 ceph-mon[104516]: pgmap v943: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:36:20.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:20 smithi064 ceph-mon[109246]: pgmap v943: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:36:21.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:21.099 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:21.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:21.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:20 smithi175 ceph-mon[101434]: pgmap v943: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T17:36:21.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 112014 111998 89.7024 59.4375 0.0189713 0.0111428 2024-08-17T17:36:21.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:21.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:21.387 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:21.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:21.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:21.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:21.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:21.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:21.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:21.840 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:21.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:21.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:21.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:22.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.137 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:22.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:22.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 112978 112962 89.3296 60.25 0.0125433 0.0111889 2024-08-17T17:36:22.238 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:22.264 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:22.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:22.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:22.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:22.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:22.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:22.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.844 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:22.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:23.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:23.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:23.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:23.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:22 smithi175 ceph-mon[101434]: pgmap v944: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:36:23.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:36:23.175290+0000 min lat: 0.00104069 max lat: 0.388376 avg lat: 0.011234 2024-08-17T17:36:23.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:36:23.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 113952 113936 88.974 60.875 0.0175781 0.011234 2024-08-17T17:36:23.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:22 smithi064 ceph-mon[104516]: pgmap v944: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:36:23.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:22 smithi064 ceph-mon[109246]: pgmap v944: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:36:23.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.274 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.419 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.643 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:23.645 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:23.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.857 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:23.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:23.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:23.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:24.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:24.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 114874 114858 88.587 57.625 0.0114592 0.0112831 2024-08-17T17:36:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:23] "GET /metrics HTTP/1.1" 200 36648 "" "Prometheus/2.51.0" 2024-08-17T17:36:24.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:24.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:24.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:24.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:24.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:24.558 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:24.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:24.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:24.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:24.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:24 smithi064 ceph-mon[104516]: pgmap v945: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-08-17T17:36:24.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:24 smithi064 ceph-mon[109246]: pgmap v945: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-08-17T17:36:24.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:24.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:24.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:24.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:25.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:25.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:24 smithi175 ceph-mon[101434]: pgmap v945: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2024-08-17T17:36:25.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 115700 115684 88.1364 51.625 0.0230099 0.0113411 2024-08-17T17:36:25.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:25.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:25.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:25.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:25.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.531 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:25.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:25.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.691 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:25.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:25.744 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:36:25.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:25.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:25.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:25.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3815396789' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:25.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:25.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:25.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3815396789' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:25.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:25.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:25.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:116392 8.e 8:7d419bfc:::benchmark_data_smithi064_242541_object116391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:25.994634+0000", 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.019040873, 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019072214000000001, 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:36:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116392 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994634+0000", 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994634+0000", 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994636+0000", 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3400000000000001e-06 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994656+0000", 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0547000000000002e-05 2024-08-17T17:36:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994672+0000", 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5480000000000001e-05 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.994679+0000", 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3420000000000002e-06 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.000909+0000", 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062302019999999998 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.001048+0000", 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013915800000000001 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.001185+0000", 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013622499999999999 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.005403+0000", 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042181450000000004 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012853+0000", 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00744972 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:116394 8.c 8:3c8f207b:::benchmark_data_smithi064_242541_object116393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:25.995150+0000", 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.018524974999999999, 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018638524, 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116394 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995150+0000", 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995150+0000", 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995151+0000", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.079e-06 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995433+0000", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028208900000000002 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995613+0000", 2024-08-17T17:36:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017988800000000001 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995622+0000", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3829999999999992e-06 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.001777+0000", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061557249999999999 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.001906+0000", 2024-08-17T17:36:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012884999999999999 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.002032+0000", 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125585 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012172+0000", 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010139881999999999 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012187+0000", 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5252999999999999e-05 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:116396 8.c 8:38843b8a:::benchmark_data_smithi064_242541_object116395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:25.995634+0000", 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.018041059000000002, 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018229366, 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116396 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995634+0000", 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995634+0000", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995635+0000", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8599999999999998e-07 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995674+0000", 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9239999999999997e-05 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995681+0000", 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9369999999999998e-06 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:25.995685+0000", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7740000000000002e-06 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.006606+0000", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010921036 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.006751+0000", 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014524000000000001 2024-08-17T17:36:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.006888+0000", 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013715600000000001 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.013696+0000", 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068083029999999999 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:116400 8.6 8:691a84c8:::benchmark_data_smithi064_242541_object116399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:26.007672+0000", 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0060028039999999996, 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062671619999999997, 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116400 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007672+0000", 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007672+0000", 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007674+0000", 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3340000000000001e-06 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007727+0000", 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3338999999999999e-05 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007743+0000", 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6379999999999999e-05 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007751+0000", 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.204e-06 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.009862+0000", 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021109129999999999 2024-08-17T17:36:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.010255+0000", 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039365199999999998 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.010771+0000", 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00051586399999999999 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:116401 8.2 e284/274 8:4ae8aa37:::benchmark_data_smithi064_242541_object116400:head v 284'7300, mlcod=284'7298)", 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:26.007826+0000", 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.005849234, 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006172528, 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007826+0000", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007826+0000", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007826+0000", 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4600000000000001e-07 2024-08-17T17:36:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007860+0000", 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3349000000000002e-05 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007866+0000", 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0390000000000001e-06 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.007870+0000", 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1629999999999998e-06 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008174+0000", 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000304481 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008193+0000", 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8464000000000001e-05 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:116406 8.2 e284/274 8:4e27540d:::benchmark_data_smithi064_242541_object116405:head v 284'7301, mlcod=284'7298)", 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:26.008935+0000", 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0047407370000000001, 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051107289999999996, 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008935+0000", 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008935+0000", 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008937+0000", 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8100000000000002e-06 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008970+0000", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2376999999999998e-05 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008981+0000", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1559999999999999e-05 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.008988+0000", 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9399999999999996e-06 2024-08-17T17:36:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012216+0000", 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032275070000000001 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012234+0000", 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8678e-05 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:116408 8.1 e284/274 8:815f0184:::benchmark_data_smithi064_242541_object116407:head v 284'7305, mlcod=284'7302)", 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:26.012275+0000", 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0014005319999999999, 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001821154, 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012275+0000", 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012275+0000", 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012276+0000", 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0869999999999999e-06 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012298+0000", 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2079999999999999e-05 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012308+0000", 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0134e-05 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012315+0000", 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7179999999999999e-06 2024-08-17T17:36:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012556+0000", 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024089000000000001 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:26.012572+0000", 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6541999999999998e-05 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:36:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 7 2024-08-17T17:36:26.050 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:36:26.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3815396789' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:26.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:25 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:26.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:26.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:26.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:26.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 116525 116509 87.6958 51.5625 0.00884433 0.0113972 2024-08-17T17:36:26.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:26.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:26.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:26.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:26.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:26.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:26.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:26.909 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:27.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:27.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[104516]: pgmap v946: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:36:27.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:27.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:27.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[104516]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:36:27.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[109246]: pgmap v946: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:36:27.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:27.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:27.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:26 smithi064 ceph-mon[109246]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:36:27.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:26 smithi175 ceph-mon[101434]: pgmap v946: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:36:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:26 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:26 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:26 smithi175 ceph-mon[101434]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:36:27.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 117420 117404 87.3155 55.9375 0.0188813 0.0114471 2024-08-17T17:36:27.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:27.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:27.288 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:27.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:27.425 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:27.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:27.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:27.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:36:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.590 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:27.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:27.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.778 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:27.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:27.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.927 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:27.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:27.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:28.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:28.044 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:28.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:28.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 15 118214 118199 86.8729 49.6875 0.0390119 0.0115047 2024-08-17T17:36:28.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:28.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:28.309 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:28.337 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:28.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:28.480 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:28.582 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:28.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:28.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:28.778 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:28.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:28.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:28 smithi064 ceph-mon[104516]: pgmap v948: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-08-17T17:36:28.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:28 smithi064 ceph-mon[109246]: pgmap v948: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-08-17T17:36:29.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:29.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:29.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:28 smithi175 ceph-mon[101434]: pgmap v948: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-08-17T17:36:29.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 119078 119062 86.49 53.9375 0.0153841 0.0115572 2024-08-17T17:36:29.185 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:29.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:29.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:29.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:29.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:29.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:29.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:29.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:29.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:29.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:29.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:29.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:29.895 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:29.896 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:29.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/332139656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:29.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/332139656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:29.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:30.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:30.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:30.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/332139656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:30.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:30.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 119947 119931 86.1202 54.3125 0.0127404 0.0116066 2024-08-17T17:36:30.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:30.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:30.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:30.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:30.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:30.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:30.576 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:30.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:30.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:30.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:30.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:30 smithi064 ceph-mon[104516]: pgmap v949: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:36:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:30 smithi064 ceph-mon[109246]: pgmap v949: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:36:30.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:31.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:31.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:30 smithi175 ceph-mon[101434]: pgmap v949: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:36:31.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:31.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 120859 120843 85.7893 57 0.00706543 0.0116503 2024-08-17T17:36:31.239 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:31.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:31.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:31.432 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:31.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:31.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:31.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:31.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.748 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:31.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:31.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:36:31.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.880 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:31.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:31.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:32.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:32.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:32.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:32.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 121707 121691 85.4209 53 0.00649522 0.0116994 2024-08-17T17:36:32.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:32.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:32.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:32.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:32.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:32.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:32.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:32.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:32.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:32.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:32.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:32.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:32.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:32 smithi064 ceph-mon[109246]: pgmap v950: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-08-17T17:36:32.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:32 smithi064 ceph-mon[104516]: pgmap v950: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-08-17T17:36:32.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:33.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:33.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:32 smithi175 ceph-mon[101434]: pgmap v950: 89 pgs: 89 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2024-08-17T17:36:33.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0165 2024-08-17T17:36:33.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 122490 2024-08-17T17:36:33.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:36:33.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 85.0469 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 36.2839 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 215.688 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 49.6875 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1360 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 580.542 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3451 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 795 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0117537 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0116503 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.388376 2024-08-17T17:36:33.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00104069 2024-08-17T17:36:33.184 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:36:33.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:33.256 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:33.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:33.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:33.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:33.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:33.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:33.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:33.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:33.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:33.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:33.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:33.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:33.935 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:34.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:34.188 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:34.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.225 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:34.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.326 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:34.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:33] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-08-17T17:36:34.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:34.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:34.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:34.662 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:34.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:34.764 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:34.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:34.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:34.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:34 smithi175 ceph-mon[101434]: pgmap v951: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-08-17T17:36:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.923 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:34.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:35.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:35.059 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:35.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:34 smithi064 ceph-mon[104516]: pgmap v951: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-08-17T17:36:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:34 smithi064 ceph-mon[109246]: pgmap v951: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-08-17T17:36:35.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:35.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:35.355 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:35.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:35.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:35.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:35.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:35.593 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:35.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:35.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:35.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:35.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:35.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:36.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:36.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:36.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:36.231 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:36.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:36.332 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:36.366 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:36.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:36.492 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:36.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:36.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:36.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:36.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:36.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:36 smithi175 ceph-mon[101434]: pgmap v952: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 865 op/s 2024-08-17T17:36:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:36.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[104516]: pgmap v952: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 865 op/s 2024-08-17T17:36:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:36.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[109246]: pgmap v952: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 865 op/s 2024-08-17T17:36:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:37.019 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:37.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:37.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:37.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:37.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:37.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:37.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:37.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:37.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:37.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-08-17T17:36:37.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:37.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:37.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:37.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:37.805 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:37.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:37.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:38.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:38.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:38.116 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:38.116 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:38.121 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:38.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:38.166 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:38.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:38.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:38.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:38.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:38.561 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:38.591 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:38.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:38.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:38.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:38.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:38.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:38 smithi175 ceph-mon[101434]: pgmap v953: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 89 B/s rd, 40 MiB/s wr, 776 op/s 2024-08-17T17:36:38.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:38 smithi064 ceph-mon[109246]: pgmap v953: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 89 B/s rd, 40 MiB/s wr, 776 op/s 2024-08-17T17:36:38.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:38 smithi064 ceph-mon[104516]: pgmap v953: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 89 B/s rd, 40 MiB/s wr, 776 op/s 2024-08-17T17:36:38.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:39.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:39.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:39.146 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:39.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:36:39.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:39.251 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:39.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:39.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:39.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:39.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:39.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:39.676 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:39.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:39.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:40.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:40.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:40.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:40.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:40.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:40.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:40.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:40.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:40.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:40.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:40.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:40.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:40.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:40.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:40.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[104516]: pgmap v954: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 783 op/s 2024-08-17T17:36:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:40.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[109246]: pgmap v954: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 783 op/s 2024-08-17T17:36:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:41.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:40 smithi175 ceph-mon[101434]: pgmap v954: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 783 op/s 2024-08-17T17:36:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:36:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:41.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:41.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:41.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.472 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:41.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:41.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:41.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:41.740 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:41.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:41.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:41.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:41.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:42.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:42.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:42.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:42.164 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:42.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:42.306 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:42.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:42.408 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:42.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:42.538 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:42.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:42.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:42.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:42.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:42 smithi064 ceph-mon[104516]: pgmap v955: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 989 op/s 2024-08-17T17:36:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:42 smithi064 ceph-mon[109246]: pgmap v955: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 989 op/s 2024-08-17T17:36:42.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.012 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:43.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:43.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:43.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:43.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:43.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:42 smithi175 ceph-mon[101434]: pgmap v955: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 989 op/s 2024-08-17T17:36:43.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:43.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:43.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:43.386 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:43.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:43.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:43.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:43.524 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:43.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:43.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.680 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:43.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:43.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:43.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:43.917 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:43.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:44.055 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:44.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:44.184 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:44.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:44.275 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:43] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:36:44.314 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:44.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:44.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.496 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:44.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:44.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:44.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:44.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:44.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:44.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:44.892 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:44.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:44 smithi064 ceph-mon[104516]: pgmap v956: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 747 op/s 2024-08-17T17:36:44.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:44 smithi064 ceph-mon[109246]: pgmap v956: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 747 op/s 2024-08-17T17:36:44.994 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:45.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:45.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:45.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:44 smithi175 ceph-mon[101434]: pgmap v956: 89 pgs: 89 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 747 op/s 2024-08-17T17:36:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:45.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:45.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:45.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:45.395 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:36:45.395 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:45.396 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:45.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:45.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:45.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:45.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:45.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:45.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:45.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:45.840 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:45.871 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2257198726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2748593784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2257198726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2748593784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:45.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:46.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:46.005 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:36:46.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2257198726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2748593784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:46.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:46.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:46.403 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:46.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:46.540 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:46.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.678 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:46.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.813 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:46.915 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[104516]: pgmap v957: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 973 op/s 2024-08-17T17:36:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[104516]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:36:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[109246]: pgmap v957: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 973 op/s 2024-08-17T17:36:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:46 smithi064 ceph-mon[109246]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:36:46.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:47.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:47.092 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:46 smithi175 ceph-mon[101434]: pgmap v957: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 973 op/s 2024-08-17T17:36:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:46 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:46 smithi175 ceph-mon[101434]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:36:47.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:47.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:47.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:47.359 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:47.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:47.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:47.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:47.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:47.621 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:47.723 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:47.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:47.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:48.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.035 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:48.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:48.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:48.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:48.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:48.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.698 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:48.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:48.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:48 smithi064 ceph-mon[104516]: pgmap v959: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:36:48.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:48 smithi064 ceph-mon[109246]: pgmap v959: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:36:48.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:49.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.112 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:49.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:49.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:48 smithi175 ceph-mon[101434]: pgmap v959: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:36:49.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:36:49.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:49.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.469 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:49.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:49.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:49.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:49.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:49.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:49.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:50.028 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:50.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:50.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:50.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:50.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:50.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:50.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:50.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:50.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:50.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:50.648 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:50.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:50.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:50.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:50.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:50 smithi064 ceph-mon[104516]: pgmap v960: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2024-08-17T17:36:50.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:50 smithi064 ceph-mon[109246]: pgmap v960: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2024-08-17T17:36:50.904 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:51.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.045 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:51.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:51.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:51.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:50 smithi175 ceph-mon[101434]: pgmap v960: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:51.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:51.308 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:51.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:51.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:51.538 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:51.564 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:51.666 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:51.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.751 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76618 6.10 6:0b484596:::benchmark_data_smithi064_185391_object76617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 582.56011579100004, 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99620523500000002, 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76618 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184811+0000", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184812+0000", 2024-08-17T17:36:51.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.389e-06 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184858+0000", 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5833e-05 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184868+0000", 2024-08-17T17:36:51.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7380000000000006e-06 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184875+0000", 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9099999999999999e-06 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184980+0000", 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104818 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.184990+0000", 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0491e-05 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176539+0000", 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99154894900000001 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176667+0000", 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012793799999999999 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.176758+0000", 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0721000000000001e-05 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180859+0000", 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041008249999999998 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180907+0000", 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8519000000000001e-05 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180943+0000", 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5848e-05 2024-08-17T17:36:51.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.180976+0000", 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3101e-05 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181016+0000", 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0154999999999998e-05 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76627 6.10 6:089e1d75:::benchmark_data_smithi064_185391_object76626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 582.54971219699996, 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98607286000000005, 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76627 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195215+0000", 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195217+0000", 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2309999999999999e-06 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195263+0000", 2024-08-17T17:36:51.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6406999999999998e-05 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195291+0000", 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8021000000000002e-05 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195302+0000", 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1068e-05 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195400+0000", 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7590000000000006e-05 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.195423+0000", 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2714e-05 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178205+0000", 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98278234499999995 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178295+0000", 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0185000000000005e-05 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:36:51.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.178394+0000", 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8790999999999997e-05 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181104+0000", 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002710329 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181147+0000", 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2651999999999998e-05 2024-08-17T17:36:51.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181195+0000", 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7511e-05 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181248+0000", 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3599e-05 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181288+0000", 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9416999999999999e-05 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:76708 6.10 6:0e16ef80:::benchmark_data_smithi064_185391_object76707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e192)", 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 582.47765083499996, 2024-08-17T17:36:51.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98798299199999995, 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 76708 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267276+0000", 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267277+0000", 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267298+0000", 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0469999999999999e-05 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267307+0000", 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8509999999999994e-06 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267314+0000", 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2759999999999997e-06 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267375+0000", 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1606000000000006e-05 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:09.267386+0000", 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.092e-05 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181547+0000", 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91416100199999994 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181662+0000", 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011430900000000001 2024-08-17T17:36:51.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.181785+0000", 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012363599999999999 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.184737+0000", 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029515589999999999 2024-08-17T17:36:51.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.187694+0000", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029570989999999999 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255151+0000", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.067457153000000006 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255214+0000", 2024-08-17T17:36:51.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3188999999999994e-05 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:10.255259+0000", 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4844e-05 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101324 6.a 6:595ddac9:::benchmark_data_smithi064_185391_object101323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 556.54933657100003, 2024-08-17T17:36:51.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99253369000000002, 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101324 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195590+0000", 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195592+0000", 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.776e-06 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195616+0000", 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3910999999999999e-05 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195629+0000", 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2731000000000001e-05 2024-08-17T17:36:51.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195637+0000", 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9589999999999995e-06 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195694+0000", 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7405999999999999e-05 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.195703+0000", 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9129999999999992e-06 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.167031+0000", 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97132766800000003 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169828+0000", 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027970429999999999 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.169985+0000", 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015776000000000001 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183860+0000", 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013874600000000001 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.187988+0000", 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041284069999999997 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.188043+0000", 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4528999999999999e-05 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.188124+0000", 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0987000000000002e-05 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101328 6.a 6:517c2a0b:::benchmark_data_smithi064_185391_object101327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 556.54733635399998, 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001870466, 2024-08-17T17:36:51.784 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101328 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:36:51.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197591+0000", 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197592+0000", 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3540000000000001e-06 2024-08-17T17:36:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197617+0000", 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4729e-05 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197635+0000", 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.806e-05 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197643+0000", 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4740000000000005e-06 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197711+0000", 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7844999999999998e-05 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.197720+0000", 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9279999999999999e-06 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174342+0000", 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97662202799999998 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174493+0000", 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150719 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.174642+0000", 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014933700000000001 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.183902+0000", 2024-08-17T17:36:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092596199999999997 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199315+0000", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015413524 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199372+0000", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6468000000000002e-05 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.199461+0000", 2024-08-17T17:36:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9380000000000001e-05 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101343 6.10 6:0d818121:::benchmark_data_smithi064_185391_object101342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 556.53568152800005, 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.007587639, 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101343 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209245+0000", 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209247+0000", 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.544e-06 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209270+0000", 2024-08-17T17:36:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2912e-05 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209288+0000", 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8263000000000001e-05 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209297+0000", 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8289999999999997e-06 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209365+0000", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8204e-05 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.209375+0000", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8810000000000008e-06 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.153546+0000", 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94417129 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.153557+0000", 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0428000000000001e-05 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211140+0000", 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057583081 2024-08-17T17:36:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211300+0000", 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015991200000000001 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.211416+0000", 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116729 2024-08-17T17:36:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215779+0000", 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043626170000000001 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:51.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215800+0000", 2024-08-17T17:36:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0744e-05 2024-08-17T17:36:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.215946+0000", 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146479 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216011+0000", 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4697999999999996e-05 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216833+0000", 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082202800000000004 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101358 6.a 6:572067d6:::benchmark_data_smithi064_185391_object101357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 556.51792609899996, 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97542105499999998, 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101358 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227001+0000", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227003+0000", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7969999999999999e-06 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227030+0000", 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7667000000000001e-05 2024-08-17T17:36:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227048+0000", 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8164000000000001e-05 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227058+0000", 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3070000000000008e-06 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227136+0000", 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8450999999999998e-05 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:35.227149+0000", 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2662e-05 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178318+0000", 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95116961899999997 2024-08-17T17:36:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178451+0000", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013217900000000001 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.178606+0000", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155582 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.184332+0000", 2024-08-17T17:36:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057253010000000003 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202269+0000", 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017937278000000001 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202329+0000", 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0217000000000003e-05 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.202422+0000", 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2831000000000006e-05 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101373 6.10 6:09660f81:::benchmark_data_smithi064_185391_object101372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 555.54981180499999, 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0313173980000001, 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101373 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195115+0000", 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195117+0000", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4279999999999999e-06 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195150+0000", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3856000000000002e-05 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195162+0000", 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1605e-05 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195170+0000", 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8140000000000001e-06 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195231+0000", 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1317999999999998e-05 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.195239+0000", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3420000000000008e-06 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.213792+0000", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018552329 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.213943+0000", 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015166299999999999 2024-08-17T17:36:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.214108+0000", 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164346 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.219035+0000", 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049269159999999999 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.219079+0000", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4691999999999998e-05 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225354+0000", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006274572 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225415+0000", 2024-08-17T17:36:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0992999999999997e-05 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.226432+0000", 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010175239999999999 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:101396 6.10 6:09c77aec:::benchmark_data_smithi064_185391_object101395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-08-17T17:36:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 555.53113650800003, 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0184236339999999, 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101396 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213790+0000", 2024-08-17T17:36:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.213791+0000", 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000001e-07 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214115+0000", 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032435899999999998 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214126+0000", 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0180000000000001e-05 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.214133+0000", 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1799999999999999e-06 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.215566+0000", 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001433024 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:36.215575+0000", 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0880000000000001e-06 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.216893+0000", 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001317784 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.217076+0000", 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000182936 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.217259+0000", 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018311099999999999 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.225311+0000", 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080522219999999995 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.226530+0000", 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012188430000000001 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.231197+0000", 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046669459999999999 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.231246+0000", 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9564999999999997e-05 2024-08-17T17:36:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:27:37.232214+0000", 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00096778000000000001 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149426 6.a 6:54a57211:::benchmark_data_smithi064_185391_object41693:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 502.42224358200002, 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.060886048, 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149426 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322683+0000", 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322684+0000", 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8800000000000001e-07 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322688+0000", 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1330000000000001e-06 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322700+0000", 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2087e-05 2024-08-17T17:36:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322708+0000", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8169999999999999e-06 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533973+0000", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21126436700000001 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533980+0000", 2024-08-17T17:36:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4789999999999999e-06 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.383499+0000", 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.849518569 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.383569+0000", 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0708000000000001e-05 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149429 6.a 6:573fa82c:::benchmark_data_smithi064_185391_object41696:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 502.422206421, 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.062352024, 2024-08-17T17:36:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149429 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:36:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322720+0000", 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322721+0000", 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.06e-07 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322724+0000", 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5440000000000001e-06 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322730+0000", 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1460000000000001e-06 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322736+0000", 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1920000000000002e-06 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.533997+0000", 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.211261265 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.534002+0000", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8339999999999998e-06 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.384980+0000", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85097835600000005 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.385072+0000", 2024-08-17T17:36:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2181000000000004e-05 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149442 6.1a 6:597706b7:::benchmark_data_smithi064_185391_object41709:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 502.42218488499998, 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.081072644, 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149442 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322742+0000", 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7100000000000002e-07 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322745+0000", 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2129999999999998e-06 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322751+0000", 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1800000000000001e-06 2024-08-17T17:36:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.322756+0000", 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8590000000000004e-06 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535270+0000", 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21251468400000001 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535279+0000", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2109999999999998e-06 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386083+0000", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.850804059 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386255+0000", 2024-08-17T17:36:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017275799999999999 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.386548+0000", 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029291700000000001 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.393934+0000", 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073854960000000001 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403675+0000", 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097411210000000002 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403730+0000", 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4724e-05 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403815+0000", 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4950999999999994e-05 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35007.0:149446 6.1a 6:585de6ab:::benchmark_data_smithi064_185391_object41713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 502.40874323499997, 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0677979660000001, 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35007", 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2206899502", 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149446 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336184+0000", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336186+0000", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.153e-06 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336192+0000", 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1510000000000001e-06 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336216+0000", 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4536e-05 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.336228+0000", 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1873e-05 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535297+0000", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19906828200000001 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:29.535302+0000", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8640000000000004e-06 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388247+0000", 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85294507799999997 2024-08-17T17:36:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388349+0000", 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102496 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.388578+0000", 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0002287 2024-08-17T17:36:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.395844+0000", 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072662600000000001 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403850+0000", 2024-08-17T17:36:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080058560000000004 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403907+0000", 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7241999999999998e-05 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:28:30.403982+0000", 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4474999999999994e-05 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96083 7.7 7:ed271dc4:::benchmark_data_smithi064_210574_object96082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.59989607699998, 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.254662196, 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:36:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96083 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145032+0000", 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9999999999999997e-07 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145053+0000", 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1299000000000001e-05 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145061+0000", 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9370000000000004e-06 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145067+0000", 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0650000000000004e-06 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145108+0000", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0983999999999999e-05 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145208+0000", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100515 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145294+0000", 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5458999999999996e-05 2024-08-17T17:36:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.157015+0000", 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011720932999999999 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.175024+0000", 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018009237000000001 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.175037+0000", 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3448999999999999e-05 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156505+0000", 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98146811099999998 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156522+0000", 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6762e-05 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.399566+0000", 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243043882 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.399693+0000", 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012686300000000001 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96119 7.7 7:e600f8d0:::benchmark_data_smithi064_210574_object96118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.54528696900002, 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2232309840000002, 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96119 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199641+0000", 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3399999999999997e-07 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199661+0000", 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0596000000000001e-05 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199669+0000", 2024-08-17T17:36:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:36:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199675+0000", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6300000000000003e-06 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202654+0000", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029795360000000001 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202665+0000", 2024-08-17T17:36:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0156999999999999e-05 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156615+0000", 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95395067300000003 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156624+0000", 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8119999999999997e-06 2024-08-17T17:36:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.402759+0000", 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2461353340000001 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.402932+0000", 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017221700000000001 2024-08-17T17:36:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.403048+0000", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116597 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.408939+0000", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058903360000000004 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409660+0000", 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072114200000000003 2024-08-17T17:36:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.418271+0000", 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086108550000000006 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.418335+0000", 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4623999999999997e-05 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422871+0000", 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045356609999999999 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96126 7.7 7:ef424558:::benchmark_data_smithi064_210574_object96125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.54193208999999, 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2244136810000001, 2024-08-17T17:36:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96126 2024-08-17T17:36:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:36:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202996+0000", 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0599999999999999e-07 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203017+0000", 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1101999999999999e-05 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203028+0000", 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1287999999999999e-05 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203035+0000", 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203087+0000", 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2159e-05 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203096+0000", 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6260000000000001e-06 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156669+0000", 2024-08-17T17:36:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95357339399999996 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156676+0000", 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3450000000000002e-06 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405915+0000", 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2492391650000001 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.406033+0000", 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118504 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.406143+0000", 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109416 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409095+0000", 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029522350000000001 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409794+0000", 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069938699999999999 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422928+0000", 2024-08-17T17:36:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013134043999999999 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422985+0000", 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6724000000000001e-05 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.427408+0000", 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044234900000000004 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:134183 7.6 7:69b70401:::benchmark_data_smithi064_210574_object35374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:31.305048+0000", 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 260.43987880999998, 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35266497699999999, 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:36:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134183 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305048+0000", 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305048+0000", 2024-08-17T17:36:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305049+0000", 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1999999999999999e-06 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305053+0000", 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7440000000000001e-06 2024-08-17T17:36:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305065+0000", 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1501000000000001e-05 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.305072+0000", 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1389999999999998e-06 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.319334+0000", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014261995 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.319428+0000", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4615000000000001e-05 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.319511+0000", 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2620000000000005e-05 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.333900+0000", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014389242 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.334066+0000", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016546200000000001 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657610+0000", 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32354428699999999 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657664+0000", 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3940999999999997e-05 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657713+0000", 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9230999999999997e-05 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:134191 7.6 7:6c13d761:::benchmark_data_smithi064_210574_object35382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:31.317144+0000", 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 260.427782783, 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34079795600000001, 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:36:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134191 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317144+0000", 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317144+0000", 2024-08-17T17:36:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317146+0000", 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4410000000000001e-06 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317150+0000", 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9459999999999997e-06 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317165+0000", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4817000000000001e-05 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.317172+0000", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.323802+0000", 2024-08-17T17:36:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006630174 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.323915+0000", 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112632 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.324008+0000", 2024-08-17T17:36:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2882999999999998e-05 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.334070+0000", 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010061679 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.344562+0000", 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010492532000000001 2024-08-17T17:36:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657851+0000", 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31328869199999998 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657906+0000", 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5435e-05 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.657942+0000", 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5732000000000003e-05 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:90932 8.6 8:6973c697:::benchmark_data_smithi064_242541_object90931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:35:59.805965+0000", 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 51.938961487, 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28531316000000001, 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 90932 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805965+0000", 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805965+0000", 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805966+0000", 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1500000000000003e-07 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805983+0000", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6583999999999999e-05 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805992+0000", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8060000000000001e-06 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.805997+0000", 2024-08-17T17:36:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8579999999999996e-06 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.810350+0000", 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043529800000000002 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.810498+0000", 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000147243 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.810636+0000", 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013874100000000001 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.817138+0000", 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065017210000000002 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.818149+0000", 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010104370000000001 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.091166+0000", 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27301757599999998 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.091231+0000", 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4490000000000001e-05 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.091279+0000", 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7908999999999998e-05 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:90939 8.6 8:6663ab88:::benchmark_data_smithi064_242541_object90938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:35:59.809720+0000", 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 51.935207349000002, 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28725721399999998, 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:36:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 90939 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809720+0000", 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809720+0000", 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:36:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809722+0000", 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0609999999999998e-06 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809747+0000", 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5213000000000001e-05 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:36:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809763+0000", 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6361000000000001e-05 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.809771+0000", 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0199999999999994e-06 2024-08-17T17:36:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.813672+0000", 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039005649999999999 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.813809+0000", 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137226 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.814003+0000", 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019385 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.821653+0000", 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076500500000000003 2024-08-17T17:36:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:35:59.821815+0000", 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162091 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.096867+0000", 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27505241600000002 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.096934+0000", 2024-08-17T17:36:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6939000000000004e-05 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:00.096977+0000", 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2422000000000001e-05 2024-08-17T17:36:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.915 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.915 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.915 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:36:51.915 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:36:51.915 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:36:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.918 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:51.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:52.019 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:52.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:52.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:52.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.189 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:52.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:52.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:52.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:52.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:52.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:52.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.616 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:52.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:52.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:52.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:52.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:52.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:52.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:52.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:53.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:53.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:52 smithi175 ceph-mon[101434]: pgmap v961: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-08-17T17:36:53.171 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:53.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:52 smithi064 ceph-mon[104516]: pgmap v961: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-08-17T17:36:53.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:52 smithi064 ceph-mon[109246]: pgmap v961: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-08-17T17:36:53.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:53.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:53.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:53.456 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:53.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:53.566 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:36:53.567 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:36:53.567 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2024-08-17T17:36:53.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:36:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.595 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:53.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:53.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:53.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:53.820 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:53.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:53.952 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:54.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:54.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:54.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:54.345 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:36:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:36:53] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:36:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:54.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:54.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:54.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:54.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:54.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.995 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:54 smithi064 ceph-mon[104516]: pgmap v962: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-08-17T17:36:54.995 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:54 smithi064 ceph-mon[109246]: pgmap v962: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-08-17T17:36:55.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:55.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:55.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:54 smithi175 ceph-mon[101434]: pgmap v962: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-08-17T17:36:55.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:55.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:55.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:55.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:55.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:55.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:36:55.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:55.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:55.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:55.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:55.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:55.880 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:55.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:55.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:55.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:56.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:56.064 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:56.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:56.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:56.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:56.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:56.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:56.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.358 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:56.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:56.442 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:56.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:56.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:56.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.641 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:56.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:56.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:56.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:56.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.908 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:56.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:57.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:57.047 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:57.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:57.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:56 smithi064 ceph-mon[104516]: pgmap v963: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-17T17:36:57.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:56 smithi064 ceph-mon[109246]: pgmap v963: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-17T17:36:57.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:56 smithi175 ceph-mon[101434]: pgmap v963: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-17T17:36:57.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:57.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:57.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:57.417 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:57.449 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:57.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:57.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:57.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:57.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:57.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:57.828 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:57.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:57.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:58.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:58.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:58.127 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:58.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:58.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:58.266 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:58.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:58.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:58.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.544 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:58.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:58.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:58.672 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:58.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:58.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:58.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:58.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:59.002 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:58 smithi064 ceph-mon[104516]: pgmap v964: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:36:59.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:58 smithi064 ceph-mon[109246]: pgmap v964: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:36:59.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:59.061 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:59.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:58 smithi175 ceph-mon[101434]: pgmap v964: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:36:59.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:59.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:59.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:59.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:59.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:59.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:59.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:59.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:59.742 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:36:59.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:59.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:36:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:59.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:59.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:36:59.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:00.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:00.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:00.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1656613212' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:59 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:36:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1656613212' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1656613212' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:36:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:00.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:00.309 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:00.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:00.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:00.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:00.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:00.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:00.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:00.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:00.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[109246]: pgmap v965: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:37:00.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[109246]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[104516]: pgmap v965: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:00.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:00 smithi064 ceph-mon[104516]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:37:00.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:00.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:00.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.997 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:00.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:01.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:01.132 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:01.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:00 smithi175 ceph-mon[101434]: pgmap v965: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:37:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:00 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:00 smithi175 ceph-mon[101434]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:37:01.234 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:01.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:01.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:01.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:01.401 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:01.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:01.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:01.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:01.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:01.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:01.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:01.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:02.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:02.200 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:02.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:02.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:02.360 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:02.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:02.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:02.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:02.658 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:02.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:02.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:02.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[104516]: pgmap v967: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:37:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:02.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[109246]: pgmap v967: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:37:02.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:02.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:02.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:02.895 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:37:02.897 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:37:02.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:02.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:03.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.063 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:03.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:02 smithi175 ceph-mon[101434]: pgmap v967: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:37:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:03.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:03.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:03.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:03.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:03.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:03.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:03.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:03.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:03.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:03.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:03.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:03.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:03 smithi064 ceph-mon[104516]: from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:03.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:03 smithi064 ceph-mon[109246]: from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:03.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:04.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:04.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:04.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:03 smithi175 ceph-mon[101434]: from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:04.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:04.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.315 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:03] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-08-17T17:37:04.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:04.442 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:04.544 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:04.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:04.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:04.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:04.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149319 8.f e286/274 8:fbd188d9:::benchmark_data_smithi064_242541_object26826:head v 286'9444, mlcod=286'9442)", 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.791710+0000", 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.01998517, 2024-08-17T17:37:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020013092, 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791710+0000", 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791710+0000", 2024-08-17T17:37:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791710+0000", 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2100000000000002e-07 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791714+0000", 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1250000000000003e-06 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791721+0000", 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8449999999999997e-06 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791726+0000", 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4000000000000002e-06 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791776+0000", 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0562000000000001e-05 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791798+0000", 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2025999999999999e-05 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149321 8.f e286/274 8:fba73977:::benchmark_data_smithi064_242541_object26828:head v 286'9445, mlcod=286'9442)", 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.791764+0000", 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.019930748000000002, 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020012583, 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791764+0000", 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791764+0000", 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791765+0000", 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8300000000000001e-07 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791770+0000", 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0579999999999997e-06 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791777+0000", 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2189999999999998e-06 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791781+0000", 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2899999999999996e-06 2024-08-17T17:37:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.801222+0000", 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094406469999999999 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.801240+0000", 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7878999999999999e-05 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149327 8.f e286/274 8:fa2704e0:::benchmark_data_smithi064_242541_object26834:head v 286'9446, mlcod=286'9442)", 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.791787+0000", 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.019907778000000001, 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020040479, 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791787+0000", 2024-08-17T17:37:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791787+0000", 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791788+0000", 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1299999999999999e-07 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791790+0000", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8499999999999998e-06 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791796+0000", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1699999999999996e-06 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.791799+0000", 2024-08-17T17:37:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8160000000000004e-06 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805500+0000", 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013700514 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805521+0000", 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1195999999999998e-05 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149332 8.b e286/274 8:d3b6090b:::benchmark_data_smithi064_242541_object26839:head v 286'9461, mlcod=286'9459)", 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.798256+0000", 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013439192000000001, 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01361891, 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798256+0000", 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798256+0000", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798257+0000", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1200000000000001e-07 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798263+0000", 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4250000000000003e-06 2024-08-17T17:37:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798275+0000", 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.186e-05 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.798282+0000", 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7399999999999998e-06 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.801172+0000", 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028900589999999999 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.801189+0000", 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7214000000000002e-05 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149333 8.f e286/274)", 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.803750+0000", 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0079455659999999994, 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081795029999999994, 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803750+0000", 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803750+0000", 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803750+0000", 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8899999999999999e-07 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803757+0000", 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0029999999999997e-06 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803767+0000", 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8530000000000004e-06 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.803774+0000", 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5509999999999996e-06 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:149343 8.7 8:ee98b5bf:::benchmark_data_smithi064_242541_object26850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.804122+0000", 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0075733140000000003, 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078498709999999996, 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149343 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804122+0000", 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804122+0000", 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804123+0000", 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2079999999999999e-06 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804126+0000", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.072e-06 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804137+0000", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1293e-05 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.804145+0000", 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7640000000000005e-06 2024-08-17T17:37:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.809820+0000", 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056744530000000003 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.809930+0000", 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110052 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:37:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.810033+0000", 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103565 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:149345 8.7 8.f1d8db67 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.805072+0000", 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0066227220000000002, 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069586500000000003, 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149345 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805072+0000", 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805072+0000", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805073+0000", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1119999999999999e-06 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805078+0000", 2024-08-17T17:37:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5719999999999996e-06 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805092+0000", 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3591000000000001e-05 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.805099+0000", 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1450000000000002e-06 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:149344 8.0 e286/274 8:0b329fde:::benchmark_data_smithi064_242541_object26851:head v 286'9224, mlcod=286'9222)", 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:04.806120+0000", 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.005574847, 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059547209999999996, 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806120+0000", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806120+0000", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806121+0000", 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0599999999999999e-07 2024-08-17T17:37:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806127+0000", 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5799999999999999e-06 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806138+0000", 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1402e-05 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.806145+0000", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5590000000000002e-06 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.810555+0000", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044105860000000002 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:04.810575+0000", 2024-08-17T17:37:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9539e-05 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2024-08-17T17:37:04.851 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:37:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:04.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:04 smithi064 ceph-mon[104516]: pgmap v968: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:37:04.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:04 smithi064 ceph-mon[109246]: pgmap v968: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:37:04.955 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:04.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:05.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:05.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:05.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:04 smithi175 ceph-mon[101434]: pgmap v968: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:37:05.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:05.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:05.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:05.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:05.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:05.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:05.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:05.657 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:05.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:05.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:05.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:37:05.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:05.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:06.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:06.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:06.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:06.186 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:06.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:06.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:06.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:06.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:06.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:06.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:06.744 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:06.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:06.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:06.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:07.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:07.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:07.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:07.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:06 smithi175 ceph-mon[101434]: pgmap v969: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-08-17T17:37:07.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:07.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:06 smithi064 ceph-mon[104516]: pgmap v969: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-08-17T17:37:07.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:06 smithi064 ceph-mon[109246]: pgmap v969: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-08-17T17:37:07.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:07.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:07.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:37:07.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:07.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:07.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:07.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:37:07.618 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:07.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:07.657 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:37:07.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:07.693 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:37:07.694 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:07.725 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:37:07.729 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:07.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:07.770 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:37:07.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:07.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3656953790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3656953790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.804 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:37:07.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:07.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:37:07.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:07.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:07.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:37:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:07.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3656953790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:08.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:08.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:08.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:08.185 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":967,"stamp":"2024-08-17T17:37:01.163876+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6533945341,"num_objects":99656,"num_object_clones":0,"num_object_copies":298968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99656,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":146477,"num_write_kb":7854402,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":10166083,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137636,"ondisk_log_size":137636,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23312848,"kb_used_data":19392332,"kb_used_omap":46,"kb_used_meta":3920145,"kb_avail":726648368,"statfs":{"total":767960285184,"available":744087928832,"internally_reserved":0,"allocated":19857747968,"data_stored":19845628812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":4014228723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,3,3,9,12,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-539361280,"num_objects":-8230,"num_object_clones":0,"num_object_copies":-24690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004459"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.693241+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:36:45.693241+0000","last_peered":"2024-08-17T17:36:45.693241+0000","last_clean":"2024-08-17T17:36:45.693241+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:36:45.693241+0000","last_undegraded":"2024-08-17T17:36:45.693241+0000","last_fullsized":"2024-08-17T17:36:45.693241+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":283,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.756856+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:36:25.756856+0000","last_peered":"2024-08-17T17:36:25.756856+0000","last_clean":"2024-08-17T17:36:25.756856+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:36:25.756856+0000","last_undegraded":"2024-08-17T17:36:25.756856+0000","last_fullsized":"2024-08-17T17:36:25.756856+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.699182+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:36:45.699182+0000","last_peered":"2024-08-17T17:36:45.699182+0000","last_clean":"2024-08-17T17:36:45.699182+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:36:45.699182+0000","last_undegraded":"2024-08-17T17:36:45.699182+0000","last_fullsized":"2024-08-17T17:36:45.699182+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":422,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.747558+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:36:59.747558+0000","last_peered":"2024-08-17T17:36:59.747558+0000","last_clean":"2024-08-17T17:36:59.747558+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:36:59.747558+0000","last_undegraded":"2024-08-17T17:36:59.747558+0000","last_fullsized":"2024-08-17T17:36:59.747558+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748539+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:36:59.748539+0000","last_peered":"2024-08-17T17:36:59.748539+0000","last_clean":"2024-08-17T17:36:59.748539+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:36:59.748539+0000","last_undegraded":"2024-08-17T17:36:59.748539+0000","last_fullsized":"2024-08-17T17:36:59.748539+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":269,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.756600+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:36:25.756600+0000","last_peered":"2024-08-17T17:36:25.756600+0000","last_clean":"2024-08-17T17:36:25.756600+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:36:25.756600+0000","last_undegraded":"2024-08-17T17:36:25.756600+0000","last_fullsized":"2024-08-17T17:36:25.756600+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":428,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.703546+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:36:45.703546+0000","last_peered":"2024-08-17T17:36:45.703546+0000","last_clean":"2024-08-17T17:36:45.703546+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:36:45.703546+0000","last_undegraded":"2024-08-17T17:36:45.703546+0000","last_fullsized":"2024-08-17T17:36:45.703546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":217,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.759544+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:36:25.759544+0000","last_peered":"2024-08-17T17:36:25.759544+0000","last_clean":"2024-08-17T17:36:25.759544+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:36:25.759544+0000","last_undegraded":"2024-08-17T17:36:25.759544+0000","last_fullsized":"2024-08-17T17:36:25.759544+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"286'9114","reported_seq":9135,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.680076+0000","last_change":"2024-08-17T17:34:48.462975+0000","last_active":"2024-08-17T17:37:00.680076+0000","last_peered":"2024-08-17T17:37:00.680076+0000","last_clean":"2024-08-17T17:37:00.680076+0000","last_became_active":"2024-08-17T17:34:48.462675+0000","last_became_peered":"2024-08-17T17:34:48.462675+0000","last_unstale":"2024-08-17T17:37:00.680076+0000","last_undegraded":"2024-08-17T17:37:00.680076+0000","last_fullsized":"2024-08-17T17:37:00.680076+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9114,"log_dups_size":0,"ondisk_log_size":9114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:53.644325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406454272,"num_objects":6202,"num_object_clones":0,"num_object_copies":18606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9114,"num_write_kb":490112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":278,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.757222+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:36:25.757222+0000","last_peered":"2024-08-17T17:36:25.757222+0000","last_clean":"2024-08-17T17:36:25.757222+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:36:25.757222+0000","last_undegraded":"2024-08-17T17:36:25.757222+0000","last_fullsized":"2024-08-17T17:36:25.757222+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698683+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:36:45.698683+0000","last_peered":"2024-08-17T17:36:45.698683+0000","last_clean":"2024-08-17T17:36:45.698683+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:36:45.698683+0000","last_undegraded":"2024-08-17T17:36:45.698683+0000","last_fullsized":"2024-08-17T17:36:45.698683+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"285'9298","reported_seq":9319,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.427972+0000","last_change":"2024-08-17T17:34:48.460978+0000","last_active":"2024-08-17T17:36:59.427972+0000","last_peered":"2024-08-17T17:36:59.427972+0000","last_clean":"2024-08-17T17:36:59.427972+0000","last_became_active":"2024-08-17T17:34:48.460684+0000","last_became_peered":"2024-08-17T17:34:48.460684+0000","last_unstale":"2024-08-17T17:36:59.427972+0000","last_undegraded":"2024-08-17T17:36:59.427972+0000","last_fullsized":"2024-08-17T17:36:59.427972+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9298,"log_dups_size":0,"ondisk_log_size":9298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:51.491253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412745728,"num_objects":6298,"num_object_clones":0,"num_object_copies":18894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9298,"num_write_kb":499072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":279,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.757538+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:36:25.757538+0000","last_peered":"2024-08-17T17:36:25.757538+0000","last_clean":"2024-08-17T17:36:25.757538+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:36:25.757538+0000","last_undegraded":"2024-08-17T17:36:25.757538+0000","last_fullsized":"2024-08-17T17:36:25.757538+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":174,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758393+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:36:25.758393+0000","last_peered":"2024-08-17T17:36:25.758393+0000","last_clean":"2024-08-17T17:36:25.758393+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:36:25.758393+0000","last_undegraded":"2024-08-17T17:36:25.758393+0000","last_fullsized":"2024-08-17T17:36:25.758393+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"286'8958","reported_seq":8994,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.841502+0000","last_change":"2024-08-17T17:34:48.460837+0000","last_active":"2024-08-17T17:36:59.841502+0000","last_peered":"2024-08-17T17:36:59.841502+0000","last_clean":"2024-08-17T17:36:59.841502+0000","last_became_active":"2024-08-17T17:34:48.460556+0000","last_became_peered":"2024-08-17T17:34:48.460556+0000","last_unstale":"2024-08-17T17:36:59.841502+0000","last_undegraded":"2024-08-17T17:36:59.841502+0000","last_fullsized":"2024-08-17T17:36:59.841502+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8958,"log_dups_size":0,"ondisk_log_size":8958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:42.107578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409206784,"num_objects":6244,"num_object_clones":0,"num_object_copies":18732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8958,"num_write_kb":486464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":296,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.757072+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:36:25.757072+0000","last_peered":"2024-08-17T17:36:25.757072+0000","last_clean":"2024-08-17T17:36:25.757072+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:36:25.757072+0000","last_undegraded":"2024-08-17T17:36:25.757072+0000","last_fullsized":"2024-08-17T17:36:25.757072+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.759900+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:36:25.759900+0000","last_peered":"2024-08-17T17:36:25.759900+0000","last_clean":"2024-08-17T17:36:25.759900+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:36:25.759900+0000","last_undegraded":"2024-08-17T17:36:25.759900+0000","last_fullsized":"2024-08-17T17:36:25.759900+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"286'9125","reported_seq":9161,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.261047+0000","last_change":"2024-08-17T17:34:48.460872+0000","last_active":"2024-08-17T17:37:00.261047+0000","last_peered":"2024-08-17T17:37:00.261047+0000","last_clean":"2024-08-17T17:37:00.261047+0000","last_became_active":"2024-08-17T17:34:48.460576+0000","last_became_peered":"2024-08-17T17:34:48.460576+0000","last_unstale":"2024-08-17T17:37:00.261047+0000","last_undegraded":"2024-08-17T17:37:00.261047+0000","last_fullsized":"2024-08-17T17:37:00.261047+0000","mapping_epoch":274,"log_start":"278'1400","ondisk_log_start":"278'1400","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:03.829227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403505152,"num_objects":6157,"num_object_clones":0,"num_object_copies":18471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9125,"num_write_kb":489024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":78,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698970+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:36:45.698970+0000","last_peered":"2024-08-17T17:36:45.698970+0000","last_clean":"2024-08-17T17:36:45.698970+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:36:45.698970+0000","last_undegraded":"2024-08-17T17:36:45.698970+0000","last_fullsized":"2024-08-17T17:36:45.698970+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":436,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748016+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:36:59.748016+0000","last_peered":"2024-08-17T17:36:59.748016+0000","last_clean":"2024-08-17T17:36:59.748016+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:36:59.748016+0000","last_undegraded":"2024-08-17T17:36:59.748016+0000","last_fullsized":"2024-08-17T17:36:59.748016+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"285'8948","reported_seq":8982,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:58.760289+0000","last_change":"2024-08-17T17:34:48.458793+0000","last_active":"2024-08-17T17:36:58.760289+0000","last_peered":"2024-08-17T17:36:58.760289+0000","last_clean":"2024-08-17T17:36:58.760289+0000","last_became_active":"2024-08-17T17:34:48.458438+0000","last_became_peered":"2024-08-17T17:34:48.458438+0000","last_unstale":"2024-08-17T17:36:58.760289+0000","last_undegraded":"2024-08-17T17:36:58.760289+0000","last_fullsized":"2024-08-17T17:36:58.760289+0000","mapping_epoch":274,"log_start":"278'800","ondisk_log_start":"278'800","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:58.357329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401211392,"num_objects":6122,"num_object_clones":0,"num_object_copies":18366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8948,"num_write_kb":482240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":450,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.709924+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:36:45.709924+0000","last_peered":"2024-08-17T17:36:45.709924+0000","last_clean":"2024-08-17T17:36:45.709924+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:36:45.709924+0000","last_undegraded":"2024-08-17T17:36:45.709924+0000","last_fullsized":"2024-08-17T17:36:45.709924+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.761121+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:36:25.761121+0000","last_peered":"2024-08-17T17:36:25.761121+0000","last_clean":"2024-08-17T17:36:25.761121+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:36:25.761121+0000","last_undegraded":"2024-08-17T17:36:25.761121+0000","last_fullsized":"2024-08-17T17:36:25.761121+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"286'9107","reported_seq":9143,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.237230+0000","last_change":"2024-08-17T17:34:48.463240+0000","last_active":"2024-08-17T17:37:00.237230+0000","last_peered":"2024-08-17T17:37:00.237230+0000","last_clean":"2024-08-17T17:37:00.237230+0000","last_became_active":"2024-08-17T17:34:48.462973+0000","last_became_peered":"2024-08-17T17:34:48.462973+0000","last_unstale":"2024-08-17T17:37:00.237230+0000","last_undegraded":"2024-08-17T17:37:00.237230+0000","last_fullsized":"2024-08-17T17:37:00.237230+0000","mapping_epoch":274,"log_start":"278'1400","ondisk_log_start":"278'1400","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:39.595929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407961600,"num_objects":6225,"num_object_clones":0,"num_object_copies":18675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9107,"num_write_kb":490624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758561+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:36:25.758561+0000","last_peered":"2024-08-17T17:36:25.758561+0000","last_clean":"2024-08-17T17:36:25.758561+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:36:25.758561+0000","last_undegraded":"2024-08-17T17:36:25.758561+0000","last_fullsized":"2024-08-17T17:36:25.758561+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":174,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758797+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:36:25.758797+0000","last_peered":"2024-08-17T17:36:25.758797+0000","last_clean":"2024-08-17T17:36:25.758797+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:36:25.758797+0000","last_undegraded":"2024-08-17T17:36:25.758797+0000","last_fullsized":"2024-08-17T17:36:25.758797+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"285'8965","reported_seq":8985,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:56.781435+0000","last_change":"2024-08-17T17:34:48.453342+0000","last_active":"2024-08-17T17:36:56.781435+0000","last_peered":"2024-08-17T17:36:56.781435+0000","last_clean":"2024-08-17T17:36:56.781435+0000","last_became_active":"2024-08-17T17:34:48.453113+0000","last_became_peered":"2024-08-17T17:34:48.453113+0000","last_unstale":"2024-08-17T17:36:56.781435+0000","last_undegraded":"2024-08-17T17:36:56.781435+0000","last_fullsized":"2024-08-17T17:36:56.781435+0000","mapping_epoch":274,"log_start":"278'800","ondisk_log_start":"278'800","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8165,"log_dups_size":0,"ondisk_log_size":8165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:39.280205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421199872,"num_objects":6427,"num_object_clones":0,"num_object_copies":19281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8965,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758285+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:36:25.758285+0000","last_peered":"2024-08-17T17:36:25.758285+0000","last_clean":"2024-08-17T17:36:25.758285+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:36:25.758285+0000","last_undegraded":"2024-08-17T17:36:25.758285+0000","last_fullsized":"2024-08-17T17:36:25.758285+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.759089+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:36:25.759089+0000","last_peered":"2024-08-17T17:36:25.759089+0000","last_clean":"2024-08-17T17:36:25.759089+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:36:25.759089+0000","last_undegraded":"2024-08-17T17:36:25.759089+0000","last_fullsized":"2024-08-17T17:36:25.759089+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"286'8870","reported_seq":8906,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.251329+0000","last_change":"2024-08-17T17:34:48.461336+0000","last_active":"2024-08-17T17:37:00.251329+0000","last_peered":"2024-08-17T17:37:00.251329+0000","last_clean":"2024-08-17T17:37:00.251329+0000","last_became_active":"2024-08-17T17:34:48.461060+0000","last_became_peered":"2024-08-17T17:34:48.461060+0000","last_unstale":"2024-08-17T17:37:00.251329+0000","last_undegraded":"2024-08-17T17:37:00.251329+0000","last_fullsized":"2024-08-17T17:37:00.251329+0000","mapping_epoch":274,"log_start":"278'1100","ondisk_log_start":"278'1100","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:41.170542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408289280,"num_objects":6230,"num_object_clones":0,"num_object_copies":18690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8870,"num_write_kb":483200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":430,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698624+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:36:45.698624+0000","last_peered":"2024-08-17T17:36:45.698624+0000","last_clean":"2024-08-17T17:36:45.698624+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:36:45.698624+0000","last_undegraded":"2024-08-17T17:36:45.698624+0000","last_fullsized":"2024-08-17T17:36:45.698624+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":418,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.694753+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:36:45.694753+0000","last_peered":"2024-08-17T17:36:45.694753+0000","last_clean":"2024-08-17T17:36:45.694753+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:36:45.694753+0000","last_undegraded":"2024-08-17T17:36:45.694753+0000","last_fullsized":"2024-08-17T17:36:45.694753+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"286'8975","reported_seq":8996,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.697523+0000","last_change":"2024-08-17T17:34:48.455278+0000","last_active":"2024-08-17T17:37:00.697523+0000","last_peered":"2024-08-17T17:37:00.697523+0000","last_clean":"2024-08-17T17:37:00.697523+0000","last_became_active":"2024-08-17T17:34:48.455018+0000","last_became_peered":"2024-08-17T17:34:48.455018+0000","last_unstale":"2024-08-17T17:37:00.697523+0000","last_undegraded":"2024-08-17T17:37:00.697523+0000","last_fullsized":"2024-08-17T17:37:00.697523+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8975,"log_dups_size":0,"ondisk_log_size":8975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:53.287262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406388736,"num_objects":6201,"num_object_clones":0,"num_object_copies":18603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8975,"num_write_kb":485632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":599,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.759337+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:36:25.759337+0000","last_peered":"2024-08-17T17:36:25.759337+0000","last_clean":"2024-08-17T17:36:25.759337+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:36:25.759337+0000","last_undegraded":"2024-08-17T17:36:25.759337+0000","last_fullsized":"2024-08-17T17:36:25.759337+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":876,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698633+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:36:45.698633+0000","last_peered":"2024-08-17T17:36:45.698633+0000","last_clean":"2024-08-17T17:36:45.698633+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:36:45.698633+0000","last_undegraded":"2024-08-17T17:36:45.698633+0000","last_fullsized":"2024-08-17T17:36:45.698633+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":413,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.692608+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:36:45.692608+0000","last_peered":"2024-08-17T17:36:45.692608+0000","last_clean":"2024-08-17T17:36:45.692608+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:36:45.692608+0000","last_undegraded":"2024-08-17T17:36:45.692608+0000","last_fullsized":"2024-08-17T17:36:45.692608+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758675+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:36:25.758675+0000","last_peered":"2024-08-17T17:36:25.758675+0000","last_clean":"2024-08-17T17:36:25.758675+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:36:25.758675+0000","last_undegraded":"2024-08-17T17:36:25.758675+0000","last_fullsized":"2024-08-17T17:36:25.758675+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"286'9138","reported_seq":9173,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.245672+0000","last_change":"2024-08-17T17:34:48.462773+0000","last_active":"2024-08-17T17:37:00.245672+0000","last_peered":"2024-08-17T17:37:00.245672+0000","last_clean":"2024-08-17T17:37:00.245672+0000","last_became_active":"2024-08-17T17:34:48.462475+0000","last_became_peered":"2024-08-17T17:34:48.462475+0000","last_unstale":"2024-08-17T17:37:00.245672+0000","last_undegraded":"2024-08-17T17:37:00.245672+0000","last_fullsized":"2024-08-17T17:37:00.245672+0000","mapping_epoch":274,"log_start":"278'1400","ondisk_log_start":"278'1400","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7738,"log_dups_size":0,"ondisk_log_size":7738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:09.376736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407240704,"num_objects":6214,"num_object_clones":0,"num_object_copies":18642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9138,"num_write_kb":491264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":662,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:36.235291+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:36:36.235291+0000","last_peered":"2024-08-17T17:36:36.235291+0000","last_clean":"2024-08-17T17:36:36.235291+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:36:36.235291+0000","last_undegraded":"2024-08-17T17:36:36.235291+0000","last_fullsized":"2024-08-17T17:36:36.235291+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":437,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.747783+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:36:59.747783+0000","last_peered":"2024-08-17T17:36:59.747783+0000","last_clean":"2024-08-17T17:36:59.747783+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:36:59.747783+0000","last_undegraded":"2024-08-17T17:36:59.747783+0000","last_fullsized":"2024-08-17T17:36:59.747783+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":434,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748736+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:36:59.748736+0000","last_peered":"2024-08-17T17:36:59.748736+0000","last_clean":"2024-08-17T17:36:59.748736+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:36:59.748736+0000","last_undegraded":"2024-08-17T17:36:59.748736+0000","last_fullsized":"2024-08-17T17:36:59.748736+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"285'9192","reported_seq":9225,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:58.765587+0000","last_change":"2024-08-17T17:34:48.457912+0000","last_active":"2024-08-17T17:36:58.765587+0000","last_peered":"2024-08-17T17:36:58.765587+0000","last_clean":"2024-08-17T17:36:58.765587+0000","last_became_active":"2024-08-17T17:34:48.457628+0000","last_became_peered":"2024-08-17T17:34:48.457628+0000","last_unstale":"2024-08-17T17:36:58.765587+0000","last_undegraded":"2024-08-17T17:36:58.765587+0000","last_fullsized":"2024-08-17T17:36:58.765587+0000","mapping_epoch":274,"log_start":"278'1000","ondisk_log_start":"278'1000","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:52.090342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414711808,"num_objects":6328,"num_object_clones":0,"num_object_copies":18984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9192,"num_write_kb":496640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":832,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.686139+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:36:45.686139+0000","last_peered":"2024-08-17T17:36:45.686139+0000","last_clean":"2024-08-17T17:36:45.686139+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:36:45.686139+0000","last_undegraded":"2024-08-17T17:36:45.686139+0000","last_fullsized":"2024-08-17T17:36:45.686139+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":426,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.765012+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:36:59.765012+0000","last_peered":"2024-08-17T17:36:59.765012+0000","last_clean":"2024-08-17T17:36:59.765012+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:36:59.765012+0000","last_undegraded":"2024-08-17T17:36:59.765012+0000","last_fullsized":"2024-08-17T17:36:59.765012+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":473,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.760064+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:36:59.760064+0000","last_peered":"2024-08-17T17:36:59.760064+0000","last_clean":"2024-08-17T17:36:59.760064+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:36:59.760064+0000","last_undegraded":"2024-08-17T17:36:59.760064+0000","last_fullsized":"2024-08-17T17:36:59.760064+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"286'9262","reported_seq":9283,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.692378+0000","last_change":"2024-08-17T17:34:48.462500+0000","last_active":"2024-08-17T17:37:00.692378+0000","last_peered":"2024-08-17T17:37:00.692378+0000","last_clean":"2024-08-17T17:37:00.692378+0000","last_became_active":"2024-08-17T17:34:48.462215+0000","last_became_peered":"2024-08-17T17:34:48.462215+0000","last_unstale":"2024-08-17T17:37:00.692378+0000","last_undegraded":"2024-08-17T17:37:00.692378+0000","last_fullsized":"2024-08-17T17:37:00.692378+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9262,"log_dups_size":0,"ondisk_log_size":9262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:31:21.306309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401997824,"num_objects":6134,"num_object_clones":0,"num_object_copies":18402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9262,"num_write_kb":492672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":999,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698964+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:36:45.698964+0000","last_peered":"2024-08-17T17:36:45.698964+0000","last_clean":"2024-08-17T17:36:45.698964+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:36:45.698964+0000","last_undegraded":"2024-08-17T17:36:45.698964+0000","last_fullsized":"2024-08-17T17:36:45.698964+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":250,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.686371+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:36:45.686371+0000","last_peered":"2024-08-17T17:36:45.686371+0000","last_clean":"2024-08-17T17:36:45.686371+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:36:45.686371+0000","last_undegraded":"2024-08-17T17:36:45.686371+0000","last_fullsized":"2024-08-17T17:36:45.686371+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":320,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.760507+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:36:59.760507+0000","last_peered":"2024-08-17T17:36:59.760507+0000","last_clean":"2024-08-17T17:36:59.760507+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:36:59.760507+0000","last_undegraded":"2024-08-17T17:36:59.760507+0000","last_fullsized":"2024-08-17T17:36:59.760507+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"286'9056","reported_seq":9091,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.824323+0000","last_change":"2024-08-17T17:34:48.464724+0000","last_active":"2024-08-17T17:36:59.824323+0000","last_peered":"2024-08-17T17:36:59.824323+0000","last_clean":"2024-08-17T17:36:59.824323+0000","last_became_active":"2024-08-17T17:34:48.464408+0000","last_became_peered":"2024-08-17T17:34:48.464408+0000","last_unstale":"2024-08-17T17:36:59.824323+0000","last_undegraded":"2024-08-17T17:36:59.824323+0000","last_fullsized":"2024-08-17T17:36:59.824323+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9056,"log_dups_size":0,"ondisk_log_size":9056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:22:49.210052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404750336,"num_objects":6176,"num_object_clones":0,"num_object_copies":18528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9056,"num_write_kb":487424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":68,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.699108+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:36:45.699108+0000","last_peered":"2024-08-17T17:36:45.699108+0000","last_clean":"2024-08-17T17:36:45.699108+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:36:45.699108+0000","last_undegraded":"2024-08-17T17:36:45.699108+0000","last_fullsized":"2024-08-17T17:36:45.699108+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":719,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.760683+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:36:59.760683+0000","last_peered":"2024-08-17T17:36:59.760683+0000","last_clean":"2024-08-17T17:36:59.760683+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:36:59.760683+0000","last_undegraded":"2024-08-17T17:36:59.760683+0000","last_fullsized":"2024-08-17T17:36:59.760683+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":443,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.758044+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:36:59.758044+0000","last_peered":"2024-08-17T17:36:59.758044+0000","last_clean":"2024-08-17T17:36:59.758044+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:36:59.758044+0000","last_undegraded":"2024-08-17T17:36:59.758044+0000","last_fullsized":"2024-08-17T17:36:59.758044+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"285'9148","reported_seq":9181,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:58.784592+0000","last_change":"2024-08-17T17:34:48.463247+0000","last_active":"2024-08-17T17:36:58.784592+0000","last_peered":"2024-08-17T17:36:58.784592+0000","last_clean":"2024-08-17T17:36:58.784592+0000","last_became_active":"2024-08-17T17:34:48.462966+0000","last_became_peered":"2024-08-17T17:34:48.462966+0000","last_unstale":"2024-08-17T17:36:58.784592+0000","last_undegraded":"2024-08-17T17:36:58.784592+0000","last_fullsized":"2024-08-17T17:36:58.784592+0000","mapping_epoch":274,"log_start":"278'1000","ondisk_log_start":"278'1000","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:28.325572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413138944,"num_objects":6304,"num_object_clones":0,"num_object_copies":18912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9148,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":440,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.760598+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:36:59.760598+0000","last_peered":"2024-08-17T17:36:59.760598+0000","last_clean":"2024-08-17T17:36:59.760598+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:36:59.760598+0000","last_undegraded":"2024-08-17T17:36:59.760598+0000","last_fullsized":"2024-08-17T17:36:59.760598+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":851,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.757116+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:36:25.757116+0000","last_peered":"2024-08-17T17:36:25.757116+0000","last_clean":"2024-08-17T17:36:25.757116+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:36:25.757116+0000","last_undegraded":"2024-08-17T17:36:25.757116+0000","last_fullsized":"2024-08-17T17:36:25.757116+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.747939+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:36:59.747939+0000","last_peered":"2024-08-17T17:36:59.747939+0000","last_clean":"2024-08-17T17:36:59.747939+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:36:59.747939+0000","last_undegraded":"2024-08-17T17:36:59.747939+0000","last_fullsized":"2024-08-17T17:36:59.747939+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"286'9060","reported_seq":9094,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.853975+0000","last_change":"2024-08-17T17:34:48.462499+0000","last_active":"2024-08-17T17:36:59.853975+0000","last_peered":"2024-08-17T17:36:59.853975+0000","last_clean":"2024-08-17T17:36:59.853975+0000","last_became_active":"2024-08-17T17:34:48.461996+0000","last_became_peered":"2024-08-17T17:34:48.461996+0000","last_unstale":"2024-08-17T17:36:59.853975+0000","last_undegraded":"2024-08-17T17:36:59.853975+0000","last_fullsized":"2024-08-17T17:36:59.853975+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9060,"log_dups_size":0,"ondisk_log_size":9060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:12.885947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409075712,"num_objects":6242,"num_object_clones":0,"num_object_copies":18726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9060,"num_write_kb":489664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":428,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.703793+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:36:45.703793+0000","last_peered":"2024-08-17T17:36:45.703793+0000","last_clean":"2024-08-17T17:36:45.703793+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:36:45.703793+0000","last_undegraded":"2024-08-17T17:36:45.703793+0000","last_fullsized":"2024-08-17T17:36:45.703793+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":949,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.760269+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:36:59.760269+0000","last_peered":"2024-08-17T17:36:59.760269+0000","last_clean":"2024-08-17T17:36:59.760269+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:36:59.760269+0000","last_undegraded":"2024-08-17T17:36:59.760269+0000","last_fullsized":"2024-08-17T17:36:59.760269+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":426,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698473+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:36:45.698473+0000","last_peered":"2024-08-17T17:36:45.698473+0000","last_clean":"2024-08-17T17:36:45.698473+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:36:45.698473+0000","last_undegraded":"2024-08-17T17:36:45.698473+0000","last_fullsized":"2024-08-17T17:36:45.698473+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"286'9138","reported_seq":9160,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:37:00.660671+0000","last_change":"2024-08-17T17:34:48.463451+0000","last_active":"2024-08-17T17:37:00.660671+0000","last_peered":"2024-08-17T17:37:00.660671+0000","last_clean":"2024-08-17T17:37:00.660671+0000","last_became_active":"2024-08-17T17:34:48.463124+0000","last_became_peered":"2024-08-17T17:34:48.463124+0000","last_unstale":"2024-08-17T17:37:00.660671+0000","last_undegraded":"2024-08-17T17:37:00.660671+0000","last_fullsized":"2024-08-17T17:37:00.660671+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9138,"log_dups_size":0,"ondisk_log_size":9138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:58:26.982388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401276952,"num_objects":6124,"num_object_clones":0,"num_object_copies":18372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6124,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9138,"num_write_kb":488321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":449,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748369+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:36:59.748369+0000","last_peered":"2024-08-17T17:36:59.748369+0000","last_clean":"2024-08-17T17:36:59.748369+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:36:59.748369+0000","last_undegraded":"2024-08-17T17:36:59.748369+0000","last_fullsized":"2024-08-17T17:36:59.748369+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":407,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.757293+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:36:25.757293+0000","last_peered":"2024-08-17T17:36:25.757293+0000","last_clean":"2024-08-17T17:36:25.757293+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:36:25.757293+0000","last_undegraded":"2024-08-17T17:36:25.757293+0000","last_fullsized":"2024-08-17T17:36:25.757293+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":226,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.697111+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:36:45.697111+0000","last_peered":"2024-08-17T17:36:45.697111+0000","last_clean":"2024-08-17T17:36:45.697111+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:36:45.697111+0000","last_undegraded":"2024-08-17T17:36:45.697111+0000","last_fullsized":"2024-08-17T17:36:45.697111+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":189,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758947+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:36:25.758947+0000","last_peered":"2024-08-17T17:36:25.758947+0000","last_clean":"2024-08-17T17:36:25.758947+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:36:25.758947+0000","last_undegraded":"2024-08-17T17:36:25.758947+0000","last_fullsized":"2024-08-17T17:36:25.758947+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":199,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.761122+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:36:25.761122+0000","last_peered":"2024-08-17T17:36:25.761122+0000","last_clean":"2024-08-17T17:36:25.761122+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:36:25.761122+0000","last_undegraded":"2024-08-17T17:36:25.761122+0000","last_fullsized":"2024-08-17T17:36:25.761122+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":425,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.757905+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:36:59.757905+0000","last_peered":"2024-08-17T17:36:59.757905+0000","last_clean":"2024-08-17T17:36:59.757905+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:36:59.757905+0000","last_undegraded":"2024-08-17T17:36:59.757905+0000","last_fullsized":"2024-08-17T17:36:59.757905+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":283,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.756984+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:36:25.756984+0000","last_peered":"2024-08-17T17:36:25.756984+0000","last_clean":"2024-08-17T17:36:25.756984+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:36:25.756984+0000","last_undegraded":"2024-08-17T17:36:25.756984+0000","last_fullsized":"2024-08-17T17:36:25.756984+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698298+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:36:45.698298+0000","last_peered":"2024-08-17T17:36:45.698298+0000","last_clean":"2024-08-17T17:36:45.698298+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:36:45.698298+0000","last_undegraded":"2024-08-17T17:36:45.698298+0000","last_fullsized":"2024-08-17T17:36:45.698298+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":390,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.692488+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:36:45.692488+0000","last_peered":"2024-08-17T17:36:45.692488+0000","last_clean":"2024-08-17T17:36:45.692488+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:36:45.692488+0000","last_undegraded":"2024-08-17T17:36:45.692488+0000","last_fullsized":"2024-08-17T17:36:45.692488+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":451,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748217+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:36:59.748217+0000","last_peered":"2024-08-17T17:36:59.748217+0000","last_clean":"2024-08-17T17:36:59.748217+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:36:59.748217+0000","last_undegraded":"2024-08-17T17:36:59.748217+0000","last_fullsized":"2024-08-17T17:36:59.748217+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":439,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698361+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:36:45.698361+0000","last_peered":"2024-08-17T17:36:45.698361+0000","last_clean":"2024-08-17T17:36:45.698361+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:36:45.698361+0000","last_undegraded":"2024-08-17T17:36:45.698361+0000","last_fullsized":"2024-08-17T17:36:45.698361+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698469+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:36:45.698469+0000","last_peered":"2024-08-17T17:36:45.698469+0000","last_clean":"2024-08-17T17:36:45.698469+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:36:45.698469+0000","last_undegraded":"2024-08-17T17:36:45.698469+0000","last_fullsized":"2024-08-17T17:36:45.698469+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":444,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698524+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:36:45.698524+0000","last_peered":"2024-08-17T17:36:45.698524+0000","last_clean":"2024-08-17T17:36:45.698524+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:36:45.698524+0000","last_undegraded":"2024-08-17T17:36:45.698524+0000","last_fullsized":"2024-08-17T17:36:45.698524+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":423,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748627+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:36:59.748627+0000","last_peered":"2024-08-17T17:36:59.748627+0000","last_clean":"2024-08-17T17:36:59.748627+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:36:59.748627+0000","last_undegraded":"2024-08-17T17:36:59.748627+0000","last_fullsized":"2024-08-17T17:36:59.748627+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":427,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.757194+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:36:59.757194+0000","last_peered":"2024-08-17T17:36:59.757194+0000","last_clean":"2024-08-17T17:36:59.757194+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:36:59.757194+0000","last_undegraded":"2024-08-17T17:36:59.757194+0000","last_fullsized":"2024-08-17T17:36:59.757194+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":441,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.692933+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:36:45.692933+0000","last_peered":"2024-08-17T17:36:45.692933+0000","last_clean":"2024-08-17T17:36:45.692933+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:36:45.692933+0000","last_undegraded":"2024-08-17T17:36:45.692933+0000","last_fullsized":"2024-08-17T17:36:45.692933+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":173,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758712+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:36:25.758712+0000","last_peered":"2024-08-17T17:36:25.758712+0000","last_clean":"2024-08-17T17:36:25.758712+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:36:25.758712+0000","last_undegraded":"2024-08-17T17:36:25.758712+0000","last_fullsized":"2024-08-17T17:36:25.758712+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":450,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.687071+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:36:45.687071+0000","last_peered":"2024-08-17T17:36:45.687071+0000","last_clean":"2024-08-17T17:36:45.687071+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:36:45.687071+0000","last_undegraded":"2024-08-17T17:36:45.687071+0000","last_fullsized":"2024-08-17T17:36:45.687071+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":428,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.747836+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:36:59.747836+0000","last_peered":"2024-08-17T17:36:59.747836+0000","last_clean":"2024-08-17T17:36:59.747836+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:36:59.747836+0000","last_undegraded":"2024-08-17T17:36:59.747836+0000","last_fullsized":"2024-08-17T17:36:59.747836+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758912+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:36:25.758912+0000","last_peered":"2024-08-17T17:36:25.758912+0000","last_clean":"2024-08-17T17:36:25.758912+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:36:25.758912+0000","last_undegraded":"2024-08-17T17:36:25.758912+0000","last_fullsized":"2024-08-17T17:36:25.758912+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":72,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.698844+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:36:45.698844+0000","last_peered":"2024-08-17T17:36:45.698844+0000","last_clean":"2024-08-17T17:36:45.698844+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:36:45.698844+0000","last_undegraded":"2024-08-17T17:36:45.698844+0000","last_fullsized":"2024-08-17T17:36:45.698844+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-17T17:36:25.758630+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:36:25.758630+0000","last_peered":"2024-08-17T17:36:25.758630+0000","last_clean":"2024-08-17T17:36:25.758630+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:36:25.758630+0000","last_undegraded":"2024-08-17T17:36:25.758630+0000","last_fullsized":"2024-08-17T17:36:25.758630+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":82,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.699109+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:36:45.699109+0000","last_peered":"2024-08-17T17:36:45.699109+0000","last_clean":"2024-08-17T17:36:45.699109+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:36:45.699109+0000","last_undegraded":"2024-08-17T17:36:45.699109+0000","last_fullsized":"2024-08-17T17:36:45.699109+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":446,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.692811+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:36:45.692811+0000","last_peered":"2024-08-17T17:36:45.692811+0000","last_clean":"2024-08-17T17:36:45.692811+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:36:45.692811+0000","last_undegraded":"2024-08-17T17:36:45.692811+0000","last_fullsized":"2024-08-17T17:36:45.692811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":447,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.693094+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:36:45.693094+0000","last_peered":"2024-08-17T17:36:45.693094+0000","last_clean":"2024-08-17T17:36:45.693094+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:36:45.693094+0000","last_undegraded":"2024-08-17T17:36:45.693094+0000","last_fullsized":"2024-08-17T17:36:45.693094+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":189,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-17T17:36:45.696790+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:36:45.696790+0000","last_peered":"2024-08-17T17:36:45.696790+0000","last_clean":"2024-08-17T17:36:45.696790+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:36:45.696790+0000","last_undegraded":"2024-08-17T17:36:45.696790+0000","last_fullsized":"2024-08-17T17:36:45.696790+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":468,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.748122+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:36:59.748122+0000","last_peered":"2024-08-17T17:36:59.748122+0000","last_clean":"2024-08-17T17:36:59.748122+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:36:59.748122+0000","last_undegraded":"2024-08-17T17:36:59.748122+0000","last_fullsized":"2024-08-17T17:36:59.748122+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6529155096,"num_objects":99628,"num_object_clones":0,"num_object_copies":298884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99628,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145354,"num_write_kb":7839361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19736440832,"data_stored":19736428616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136454,"ondisk_log_size":136454,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201681,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2553148,"kb_used_data":2070644,"kb_used_omap":1,"kb_used_meta":482302,"kb_avail":91192004,"statfs":{"total":95995035648,"available":93380612096,"internally_reserved":0,"allocated":2120339456,"data_stored":2118807179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":493877707},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922178,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2215400,"kb_used_data":2015392,"kb_used_omap":7,"kb_used_meta":199992,"kb_avail":91529752,"statfs":{"total":95995035648,"available":93726466048,"internally_reserved":0,"allocated":2063761408,"data_stored":2062248553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":204792502},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085702,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3357240,"kb_used_data":3179948,"kb_used_omap":7,"kb_used_meta":177272,"kb_avail":90387912,"statfs":{"total":95995035648,"available":92557221888,"internally_reserved":0,"allocated":3256266752,"data_stored":3254746111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":181526969},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986591,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2623408,"kb_used_data":2060712,"kb_used_omap":1,"kb_used_meta":562686,"kb_avail":91121744,"statfs":{"total":95995035648,"available":93308665856,"internally_reserved":0,"allocated":2110169088,"data_stored":2108648469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":576190918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435483,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3045916,"kb_used_data":2414484,"kb_used_omap":5,"kb_used_meta":631418,"kb_avail":90699236,"statfs":{"total":95995035648,"available":92876017664,"internally_reserved":0,"allocated":2472431616,"data_stored":2470926677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":646572105},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599009,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3142944,"kb_used_data":2461528,"kb_used_omap":7,"kb_used_meta":681400,"kb_avail":90602208,"statfs":{"total":95995035648,"available":92776660992,"internally_reserved":0,"allocated":2520604672,"data_stored":2519095010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":697753773},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762536,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3326688,"kb_used_data":2781336,"kb_used_omap":7,"kb_used_meta":545336,"kb_avail":90418464,"statfs":{"total":95995035648,"available":92588507136,"internally_reserved":0,"allocated":2848088064,"data_stored":2846585797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":558424481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926062,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3048104,"kb_used_data":2408288,"kb_used_omap":7,"kb_used_meta":639736,"kb_avail":90697048,"statfs":{"total":95995035648,"available":92873777152,"internally_reserved":0,"allocated":2466086912,"data_stored":2464571016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":655090268},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2452422656,"data_stored":2452422656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2843279360,"data_stored":2843279360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2510229504,"data_stored":2510225432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2457141248,"data_stored":2457141248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2095120384,"data_stored":2095120384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3235385344,"data_stored":3235381272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2045902848,"data_stored":2045902848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2096959488,"data_stored":2096955416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:37:08.189 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:37:08.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_3 pg_num 26 2024-08-17T17:37:08.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:08.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:08.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:08.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:08.571 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:08.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:08.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:08.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:08.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:08 smithi064 ceph-mon[104516]: pgmap v970: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-08-17T17:37:08.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:08 smithi064 ceph-mon[109246]: pgmap v970: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-08-17T17:37:08.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:08 smithi175 ceph-mon[101434]: pgmap v970: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-08-17T17:37:08.931 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:08.943 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:08.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:09.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:09.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:09.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:09.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:09.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:09.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:09.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.589 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:09.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:09.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:09.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:09.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:09.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:09.862 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:09.888 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:10.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:10.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:10.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:10.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:10.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:10.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:10.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:10.497 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:10.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.595 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:10.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:10.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:10.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:10.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:10.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:10.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:10 smithi064 ceph-mon[104516]: pgmap v971: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T17:37:10.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:10 smithi064 ceph-mon[109246]: pgmap v971: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T17:37:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:10.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.033 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:10 smithi175 ceph-mon[101434]: pgmap v971: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T17:37:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.179 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:11.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:11.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:11.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.469 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:11.612 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:11.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:11.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:11.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.886 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:11.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:11.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:12.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:12.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:12.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:12.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:12.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:12.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:12.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:12.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:12.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.692 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:12.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:13.061 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:12 smithi064 ceph-mon[104516]: pgmap v972: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-08-17T17:37:13.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:12 smithi064 ceph-mon[109246]: pgmap v972: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-08-17T17:37:13.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:13.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:12 smithi175 ceph-mon[101434]: pgmap v972: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-08-17T17:37:13.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:13.225 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:13.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:13.368 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:13.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:13.470 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:13.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:13.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:13.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:13.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:13.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:13.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:13.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:14.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:14.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:14.065 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:13] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-08-17T17:37:14.151 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:14.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:14.238 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:14.238 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:14.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:14.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:14.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:14.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:14.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:14.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:14.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:14.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:14.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:14.861 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:14 smithi064 ceph-mon[104516]: pgmap v973: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-08-17T17:37:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:14 smithi064 ceph-mon[109246]: pgmap v973: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-08-17T17:37:14.962 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:14.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:15.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:15.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:15.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:14 smithi175 ceph-mon[101434]: pgmap v973: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-08-17T17:37:15.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:15.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:15.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:37:15.367 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:15.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:15.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:15.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:15.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.661 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:15.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:15.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:15.777 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:37:15.783 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:15.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:15.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3013410599' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:15.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:15 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:15.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3013410599' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:15.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:15 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:15.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:15.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:16.073 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:16.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:16.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3013410599' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:16.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:15 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:16.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:16.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:16.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:16.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:16.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:16.519 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:16.621 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:16.646 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:16.712 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:16.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:16.881 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:16.909 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[104516]: pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:37:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[104516]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[109246]: pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:16 smithi064 ceph-mon[109246]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:37:17.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:17.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:17.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:17.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:16 smithi175 ceph-mon[101434]: pgmap v974: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:37:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:16 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:16 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:37:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:17.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:16 smithi175 ceph-mon[101434]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:37:17.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:17.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:17.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:17.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:17.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:17.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:17.471 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:17.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:17.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:17.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:17.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:17.749 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:17.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:17.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:17.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:18.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:18.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:18.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:18.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:18.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:18.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:18.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:18.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:18.601 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:18.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:37:18.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:18.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:18.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:18.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:18.867 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:18.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:19.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:19.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:19.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:18 smithi175 ceph-mon[101434]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:37:19.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:18 smithi064 ceph-mon[104516]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:37:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:18 smithi064 ceph-mon[109246]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:37:19.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:19.289 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:19.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:19.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:19.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:19.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:19.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:19.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:19.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:19.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.990 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:20.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:20.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:20.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:20.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:20.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:20.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:20.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:20.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:20.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:20.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:20.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:20.766 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:20.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:20.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:20 smithi064 ceph-mon[104516]: pgmap v977: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-17T17:37:20.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:20 smithi064 ceph-mon[109246]: pgmap v977: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-17T17:37:20.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:20.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:21.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:21.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:21.106 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:37:21.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:21.139 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:37:21.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:20 smithi175 ceph-mon[101434]: pgmap v977: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-17T17:37:21.166 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:21.172 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:37:21.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:21.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:21.209 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:37:21.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:21.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:37:21.243 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:21.271 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:37:21.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:21.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:21.305 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:37:21.306 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:21.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:21.344 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:37:21.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:21.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:21.484 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:21.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:21.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:21.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:21.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:21.852 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:22.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2624549980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:22.163 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:22.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2624549980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:22.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2624549980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:22.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:22.305 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:22.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:22.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:22.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:22.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:22.706 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:22.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:22.837 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 8 pg_num to 26 2024-08-17T17:37:22.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:22.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:22.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:22.988 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:23.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:23.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:22 smithi175 ceph-mon[101434]: pgmap v978: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:37:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:22 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/258643932' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[104516]: pgmap v978: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:37:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/258643932' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[109246]: pgmap v978: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:37:23.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/258643932' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:37:23.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:23.391 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:23.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:23.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:23.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:23.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:23.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:23.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:23.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:23.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:23.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[104516]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[104516]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[109246]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[109246]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:37:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:37:23.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:23.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:24.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.095 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:24.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:23 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:37:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:23 smithi175 ceph-mon[101434]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:37:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:23 smithi175 ceph-mon[101434]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:37:24.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:37:24.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:23] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-08-17T17:37:24.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:24.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:24.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:24.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:24.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:24.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:24.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:24.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:24.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:24.731 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:24.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:24.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:24.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:37:24.963 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:24.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:25.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:25.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:37:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:24 smithi175 ceph-mon[101434]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:37:25.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:37:25.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:24 smithi064 ceph-mon[104516]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:37:25.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:37:25.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:24 smithi064 ceph-mon[109246]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:37:25.219 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:25.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.484 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:37:25.484 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:37:25.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:37:25.484 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96085 7.b 7:d7850583:::benchmark_data_smithi064_210574_object96084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.152288+0000", 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.09049672700002, 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2686717939999999, 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96085 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152288+0000", 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152288+0000", 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152290+0000", 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9429999999999999e-06 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152320+0000", 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0287999999999999e-05 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152333+0000", 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2941e-05 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.152343+0000", 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4669999999999993e-06 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.157000+0000", 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046577930000000003 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.157012+0000", 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1761e-05 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157221+0000", 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0002092380000001 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157234+0000", 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2506e-05 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405204+0000", 2024-08-17T17:37:25.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2479699609999999 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405373+0000", 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016961600000000001 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405547+0000", 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017351899999999999 2024-08-17T17:37:25.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414305+0000", 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087580550000000007 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414392+0000", 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7206000000000005e-05 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.417779+0000", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033865929999999998 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.417846+0000", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7700999999999994e-05 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.420960+0000", 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031132059999999999 2024-08-17T17:37:25.493 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96089 7.d 7:bee52d32:::benchmark_data_smithi064_210574_object96088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.161963+0000", 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.08082116399999, 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2577383539999998, 2024-08-17T17:37:25.494 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96089 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.495 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.161963+0000", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.161963+0000", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.161965+0000", 2024-08-17T17:37:25.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4670000000000001e-06 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.161988+0000", 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3383e-05 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.161997+0000", 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9120000000000004e-06 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162004+0000", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4740000000000002e-06 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162111+0000", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106913 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162124+0000", 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3604999999999999e-05 2024-08-17T17:37:25.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157374+0000", 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99524938100000004 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157384+0000", 2024-08-17T17:37:25.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9399999999999997e-06 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405117+0000", 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2477336509999999 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405231+0000", 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114016 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405360+0000", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000128489 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.413995+0000", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086352919999999993 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414011+0000", 2024-08-17T17:37:25.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6433999999999999e-05 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.416410+0000", 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023988830000000001 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.416452+0000", 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1699999999999997e-05 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.419702+0000", 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032498140000000002 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96090 7.d 7:bd5828ed:::benchmark_data_smithi064_210574_object96089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.162018+0000", 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.08076616300002, 2024-08-17T17:37:25.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2882647810000001, 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96090 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162018+0000", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162018+0000", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162019+0000", 2024-08-17T17:37:25.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5199999999999997e-07 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162065+0000", 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6347000000000002e-05 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162073+0000", 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3869999999999997e-06 2024-08-17T17:37:25.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162078+0000", 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4739999999999996e-06 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162152+0000", 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3535000000000007e-05 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.162160+0000", 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5730000000000008e-06 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157439+0000", 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.995278459 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157454+0000", 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5668999999999999e-05 2024-08-17T17:37:25.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.410950+0000", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2534951839999999 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.411126+0000", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017650200000000001 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.411335+0000", 2024-08-17T17:37:25.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020883300000000001 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.419792+0000", 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084568290000000008 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.419811+0000", 2024-08-17T17:37:25.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9675000000000001e-05 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.440797+0000", 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020985772999999999 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.440857+0000", 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0173999999999997e-05 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.450283+0000", 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094258150000000006 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96113 7.b 7:d1ca955f:::benchmark_data_smithi064_210574_object96112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.193707+0000", 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.049077458, 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2517907519999998, 2024-08-17T17:37:25.512 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96113 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193707+0000", 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193707+0000", 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193709+0000", 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2469999999999998e-06 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193732+0000", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.247e-05 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193754+0000", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.209e-05 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193762+0000", 2024-08-17T17:37:25.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0690000000000002e-06 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193856+0000", 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4277000000000006e-05 2024-08-17T17:37:25.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.193867+0000", 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0487e-05 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157322+0000", 2024-08-17T17:37:25.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96345504400000004 2024-08-17T17:37:25.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157331+0000", 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.411560+0000", 2024-08-17T17:37:25.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.254229198 2024-08-17T17:37:25.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.411713+0000", 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153008 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.411823+0000", 2024-08-17T17:37:25.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109993 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424420+0000", 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012596326999999999 2024-08-17T17:37:25.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.427972+0000", 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035521939999999998 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.442039+0000", 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014067603999999999 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.442096+0000", 2024-08-17T17:37:25.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6453999999999999e-05 2024-08-17T17:37:25.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.445498+0000", 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034020180000000001 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96121 7.b 7:dc8a205d:::benchmark_data_smithi064_210574_object96120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.201637+0000", 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.04114788499999, 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2476501839999998, 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.528 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96121 2024-08-17T17:37:25.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.530 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201637+0000", 2024-08-17T17:37:25.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201637+0000", 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201638+0000", 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.711e-06 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201664+0000", 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5449e-05 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201681+0000", 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6928000000000001e-05 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201690+0000", 2024-08-17T17:37:25.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6889999999999993e-06 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201771+0000", 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1186000000000005e-05 2024-08-17T17:37:25.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.201784+0000", 2024-08-17T17:37:25.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3175e-05 2024-08-17T17:37:25.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157355+0000", 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95557136399999998 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157364+0000", 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4540000000000007e-06 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414562+0000", 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.257198297 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414708+0000", 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145785 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414901+0000", 2024-08-17T17:37:25.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019311999999999999 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424459+0000", 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095585170000000007 2024-08-17T17:37:25.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.427979+0000", 2024-08-17T17:37:25.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035192489999999999 2024-08-17T17:37:25.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.445557+0000", 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017578812999999999 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.445620+0000", 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2584999999999998e-05 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.449287+0000", 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036668619999999999 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.546 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96124 7.d 7:bff6fc13:::benchmark_data_smithi064_210574_object96123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.546 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.202604+0000", 2024-08-17T17:37:25.546 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.04018060800001, 2024-08-17T17:37:25.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2535066759999998, 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96124 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202604+0000", 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202604+0000", 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202605+0000", 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1999999999999999e-07 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202655+0000", 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0241999999999998e-05 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202665+0000", 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0261000000000001e-05 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202672+0000", 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8870000000000003e-06 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202829+0000", 2024-08-17T17:37:25.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015705999999999999 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202840+0000", 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0709e-05 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157490+0000", 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95464992400000004 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157498+0000", 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9310000000000008e-06 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414037+0000", 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.256539413 2024-08-17T17:37:25.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414145+0000", 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010810399999999999 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.414242+0000", 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6446000000000005e-05 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.419936+0000", 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056946499999999999 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.432999+0000", 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013062696 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.450425+0000", 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017425955999999999 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.450494+0000", 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8703999999999998e-05 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.456111+0000", 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056169729999999999 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96132 7.b 7:df938af3:::benchmark_data_smithi064_210574_object96131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.212362+0000", 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.03042228800001, 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2405910850000001, 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.558 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96132 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212362+0000", 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212362+0000", 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212364+0000", 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.73e-06 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212413+0000", 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8473000000000002e-05 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212433+0000", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0635000000000001e-05 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212441+0000", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3950000000000003e-06 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212503+0000", 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2785000000000003e-05 2024-08-17T17:37:25.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.212515+0000", 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1433e-05 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157401+0000", 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94488654999999999 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157411+0000", 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7869999999999996e-06 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.421026+0000", 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.263615264 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.421187+0000", 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016038599999999999 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.421327+0000", 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140154 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424506+0000", 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003178775 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.427987+0000", 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034815599999999999 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.449368+0000", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021380822000000001 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.449448+0000", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9832000000000004e-05 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.452953+0000", 2024-08-17T17:37:25.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003505504 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96134 7.d 7:b98415f0:::benchmark_data_smithi064_210574_object96133:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.215528+0000", 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.02725694399999, 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2446516449999998, 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.567 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96134 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215528+0000", 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215528+0000", 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215529+0000", 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.443e-06 2024-08-17T17:37:25.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215547+0000", 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.804e-05 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215561+0000", 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3390999999999999e-05 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215567+0000", 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215634+0000", 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6168999999999996e-05 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.215643+0000", 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8290000000000003e-06 2024-08-17T17:37:25.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157534+0000", 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94189087500000002 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157543+0000", 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5529999999999993e-06 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.419966+0000", 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2624227139999999 2024-08-17T17:37:25.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.420096+0000", 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129927 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:37:25.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.420251+0000", 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015561899999999999 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.432979+0000", 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012728276 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.436821+0000", 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038417529999999998 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.456229+0000", 2024-08-17T17:37:25.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019407954000000002 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.456296+0000", 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7189999999999999e-05 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.460179+0000", 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038830029999999999 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.579 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96135 7.b 7:dc9d33a6:::benchmark_data_smithi064_210574_object96134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.216526+0000", 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 350.026258816, 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2399428779999999, 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96135 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216526+0000", 2024-08-17T17:37:25.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216526+0000", 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216527+0000", 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0800000000000004e-07 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216544+0000", 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7280999999999999e-05 2024-08-17T17:37:25.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216552+0000", 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3680000000000002e-06 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216557+0000", 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9989999999999999e-06 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216623+0000", 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5957000000000006e-05 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.216630+0000", 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1539999999999996e-06 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157461+0000", 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94083029500000004 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.157469+0000", 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7029999999999995e-06 2024-08-17T17:37:25.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424543+0000", 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2670731239999999 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424694+0000", 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151273 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.424843+0000", 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014936699999999999 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.428115+0000", 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032714430000000002 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.428179+0000", 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4443999999999995e-05 2024-08-17T17:37:25.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.453022+0000", 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024843264 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.453106+0000", 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4116999999999994e-05 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.456469+0000", 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003362281 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:134219 7.d 7:b2f7e8e2:::benchmark_data_smithi064_210574_object35410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:31.354971+0000", 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 293.88781343900001, 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33932909300000003, 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.590 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134219 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354971+0000", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354971+0000", 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354973+0000", 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3209999999999999e-06 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354976+0000", 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.692e-06 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354984+0000", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0879999999999995e-06 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.354989+0000", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4819999999999996e-06 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.355029+0000", 2024-08-17T17:37:25.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0287000000000002e-05 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.355132+0000", 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010328299999999999 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.355226+0000", 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3354000000000001e-05 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.358491+0000", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003265273 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.365923+0000", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074319520000000003 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.691108+0000", 2024-08-17T17:37:25.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32518518600000001 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.691162+0000", 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3888000000000003e-05 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.694300+0000", 2024-08-17T17:37:25.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003138287 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:145570 8.3 8:cbbdcef9:::benchmark_data_smithi064_242541_object23077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:59.895158+0000", 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.347627086999999, 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33844074699999999, 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.597 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145570 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895158+0000", 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895158+0000", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895159+0000", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8500000000000001e-06 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895165+0000", 2024-08-17T17:37:25.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6380000000000001e-06 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895178+0000", 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2529e-05 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895185+0000", 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9330000000000001e-06 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895304+0000", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011877799999999999 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895435+0000", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013026800000000001 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.895543+0000", 2024-08-17T17:37:25.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108619 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.910507+0000", 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014964244 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.911455+0000", 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094795899999999995 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230048+0000", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31859277200000002 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230269+0000", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000221108 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.233598+0000", 2024-08-17T17:37:25.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033290490000000002 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:145580 8.a 8:5b3045c7:::benchmark_data_smithi064_242541_object23087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:59.904871+0000", 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.337913799999999, 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34056413299999999, 2024-08-17T17:37:25.604 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145580 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904871+0000", 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904871+0000", 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904873+0000", 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7039999999999999e-06 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904879+0000", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.489e-06 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904902+0000", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2512000000000001e-05 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.904910+0000", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3580000000000003e-06 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.905158+0000", 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000247638 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.905432+0000", 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027399999999999999 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.905548+0000", 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116549 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230788+0000", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.325240209 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230820+0000", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2113000000000002e-05 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245058+0000", 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014237469000000001 2024-08-17T17:37:25.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245122+0000", 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4368000000000004e-05 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245435+0000", 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031272400000000001 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:145581 8.a 8:5507c9c2:::benchmark_data_smithi064_242541_object23088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:36:59.907521+0000", 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.335264107, 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33810399299999999, 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 145581 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907521+0000", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907521+0000", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907522+0000", 2024-08-17T17:37:25.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5719999999999999e-06 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907527+0000", 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6079999999999998e-06 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907540+0000", 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2884999999999999e-05 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.907548+0000", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9880000000000004e-06 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.910580+0000", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030325500000000002 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.910695+0000", 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115194 2024-08-17T17:37:25.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:36:59.910814+0000", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118472 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230851+0000", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32003763200000002 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.230894+0000", 2024-08-17T17:37:25.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2466999999999998e-05 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245545+0000", 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014650942 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245589+0000", 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4097999999999997e-05 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:00.245625+0000", 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5584999999999999e-05 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:150645 8.1 8:84580deb:::benchmark_data_smithi064_242541_object28152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:06.538953+0000", 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.703832016, 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33586432900000002, 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.617 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 150645 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538953+0000", 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538953+0000", 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538954+0000", 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.858e-06 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538960+0000", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2739999999999998e-06 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538976+0000", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5990000000000001e-05 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.538984+0000", 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8320000000000006e-06 2024-08-17T17:37:25.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.539041+0000", 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7768e-05 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.539163+0000", 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012192399999999999 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:37:25.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.539286+0000", 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012302299999999999 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.546152+0000", 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068652169999999998 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.546198+0000", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6056000000000001e-05 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.874686+0000", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32848812100000002 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.874764+0000", 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7923000000000006e-05 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:06.874817+0000", 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3343000000000001e-05 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164895 8.a 8:5a2a81af:::benchmark_data_smithi064_242541_object42402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:23.863335+0000", 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.379449454, 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.356591453, 2024-08-17T17:37:25.624 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164895 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863335+0000", 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863335+0000", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863337+0000", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7460000000000001e-06 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863341+0000", 2024-08-17T17:37:25.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7120000000000002e-06 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863350+0000", 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6509999999999996e-06 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863356+0000", 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1569999999999997e-06 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863531+0000", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017445899999999999 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863664+0000", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013328099999999999 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.863788+0000", 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012398199999999999 2024-08-17T17:37:25.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.868018+0000", 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042298819999999999 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.868463+0000", 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000444771 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:37:25.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.219779+0000", 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35131656100000003 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.219834+0000", 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4580999999999998e-05 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.219927+0000", 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2670000000000006e-05 2024-08-17T17:37:25.630 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164880 8.13 8:cf2a433e:::benchmark_data_smithi064_242541_object42387:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.038951+0000", 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.203833471, 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91981383999999999, 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.631 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164880 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038951+0000", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038951+0000", 2024-08-17T17:37:25.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038953+0000", 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.289e-06 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038961+0000", 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9670000000000001e-06 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038983+0000", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1290000000000001e-05 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038990+0000", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4030000000000001e-06 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121940+0000", 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082950253000000002 2024-08-17T17:37:25.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121947+0000", 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9e-06 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.958654+0000", 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83670689799999998 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.958765+0000", 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011084 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164882 8.13 8:cd4bae4c:::benchmark_data_smithi064_242541_object42389:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.039004+0000", 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.203780257, 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96210931200000005, 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.636 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164882 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039004+0000", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039004+0000", 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039005+0000", 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3700000000000001e-07 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039007+0000", 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0090000000000002e-06 2024-08-17T17:37:25.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039014+0000", 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8889999999999996e-06 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039017+0000", 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.179e-06 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121966+0000", 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082949505000000007 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121972+0000", 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4480000000000002e-06 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.969229+0000", 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84725751800000004 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.969484+0000", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025462299999999999 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:25.001016+0000", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031531970999999999 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:25.001114+0000", 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7832999999999997e-05 2024-08-17T17:37:25.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164884 8.13 8:cf4152fc:::benchmark_data_smithi064_242541_object42391:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.039021+0000", 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.2037636629999999, 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96196359200000003, 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164884 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039021+0000", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039021+0000", 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039021+0000", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3999999999999998e-07 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039023+0000", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3179999999999999e-06 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039029+0000", 2024-08-17T17:37:25.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7499999999999997e-06 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039032+0000", 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8279999999999999e-06 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121990+0000", 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082957989999999995 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121995+0000", 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.237e-06 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.969515+0000", 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84751943799999996 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.970158+0000", 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064364499999999998 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:25.000860+0000", 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030701381 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:25.000985+0000", 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012476499999999999 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.647 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164886 8.11 8:8b6964e9:::benchmark_data_smithi064_242541_object42393:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.039036+0000", 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.20374903, 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91520442000000002, 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164886 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039036+0000", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039036+0000", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039036+0000", 2024-08-17T17:37:25.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3799999999999999e-07 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039037+0000", 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.167e-06 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039040+0000", 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2389999999999998e-06 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039043+0000", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.819e-06 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.114022+0000", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074978990999999995 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.114032+0000", 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9909999999999997e-06 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.949369+0000", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83533685000000002 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.949508+0000", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013919599999999999 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.954163+0000", 2024-08-17T17:37:25.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046548609999999997 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.954240+0000", 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7068000000000001e-05 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164896 8.3 8:c3871f98:::benchmark_data_smithi064_242541_object42403:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.039046+0000", 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.203738427, 2024-08-17T17:37:25.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93015840400000005, 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164896 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039046+0000", 2024-08-17T17:37:25.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039046+0000", 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039046+0000", 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.03e-07 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9399999999999993e-07 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039054+0000", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5459999999999997e-06 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.039057+0000", 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6759999999999999e-06 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121707+0000", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.082650162999999999 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.121719+0000", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2019e-05 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.939221+0000", 2024-08-17T17:37:25.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81750237699999995 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.939360+0000", 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013861900000000001 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.969133+0000", 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.029772836 2024-08-17T17:37:25.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.969205+0000", 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1971000000000006e-05 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:25.659 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:37:25.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.661 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:25.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:25.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:25.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:25.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:26.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:26.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:26.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:25 smithi175 ceph-mon[101434]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:37:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:25 smithi175 ceph-mon[101434]: pgmap v983: 99 pgs: 1 peering, 9 unknown, 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:37:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:25 smithi175 ceph-mon[101434]: 8.0 scrub starts 2024-08-17T17:37:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[104516]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:37:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:26.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[104516]: pgmap v983: 99 pgs: 1 peering, 9 unknown, 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[104516]: 8.0 scrub starts 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[109246]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[109246]: pgmap v983: 99 pgs: 1 peering, 9 unknown, 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:37:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:25 smithi064 ceph-mon[109246]: 8.0 scrub starts 2024-08-17T17:37:26.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:26.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:26.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:26.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:26.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:26.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:26.581 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:26.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:37:26.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:26.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:26.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:26.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:26.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:26.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:26.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:27.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:27.114 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:27.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[104516]: 8.7 scrub starts 2024-08-17T17:37:27.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[104516]: 8.6 scrub starts 2024-08-17T17:37:27.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[104516]: 8.0 scrub ok 2024-08-17T17:37:27.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[104516]: 8.10 deep-scrub starts 2024-08-17T17:37:27.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[109246]: 8.7 scrub starts 2024-08-17T17:37:27.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[109246]: 8.6 scrub starts 2024-08-17T17:37:27.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[109246]: 8.0 scrub ok 2024-08-17T17:37:27.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:26 smithi064 ceph-mon[109246]: 8.10 deep-scrub starts 2024-08-17T17:37:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:26 smithi175 ceph-mon[101434]: 8.7 scrub starts 2024-08-17T17:37:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:26 smithi175 ceph-mon[101434]: 8.6 scrub starts 2024-08-17T17:37:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:26 smithi175 ceph-mon[101434]: 8.0 scrub ok 2024-08-17T17:37:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:26 smithi175 ceph-mon[101434]: 8.10 deep-scrub starts 2024-08-17T17:37:27.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:27.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:27.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:27.391 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:27.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:27.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:27.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:27.664 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:27.766 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:27.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:27.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:27.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:28.031 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:28.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:28.085 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:28.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:28.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.6 scrub ok 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.7 scrub ok 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.1 deep-scrub starts 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.16 scrub starts 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: pgmap v984: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.2 scrub starts 2024-08-17T17:37:28.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:27 smithi175 ceph-mon[101434]: 8.2 scrub ok 2024-08-17T17:37:28.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.6 scrub ok 2024-08-17T17:37:28.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.7 scrub ok 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.1 deep-scrub starts 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.16 scrub starts 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: pgmap v984: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.2 scrub starts 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[104516]: 8.2 scrub ok 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.6 scrub ok 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.7 scrub ok 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.1 deep-scrub starts 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.16 scrub starts 2024-08-17T17:37:28.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: pgmap v984: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:28.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.2 scrub starts 2024-08-17T17:37:28.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:27 smithi064 ceph-mon[109246]: 8.2 scrub ok 2024-08-17T17:37:28.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:28.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:28.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:28.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:28.465 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:28.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:28.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:28.685 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:37:28.685 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:37:28.685 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2024-08-17T17:37:28.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:37:28.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:28.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:28.886 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:28.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:28.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:29.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:29.104 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[109246]: 8.16 scrub ok 2024-08-17T17:37:29.104 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[109246]: 8.17 scrub starts 2024-08-17T17:37:29.104 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[109246]: 8.17 scrub ok 2024-08-17T17:37:29.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[104516]: 8.16 scrub ok 2024-08-17T17:37:29.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[104516]: 8.17 scrub starts 2024-08-17T17:37:29.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:28 smithi064 ceph-mon[104516]: 8.17 scrub ok 2024-08-17T17:37:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:29.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:28 smithi175 ceph-mon[101434]: 8.16 scrub ok 2024-08-17T17:37:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:28 smithi175 ceph-mon[101434]: 8.17 scrub starts 2024-08-17T17:37:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:28 smithi175 ceph-mon[101434]: 8.17 scrub ok 2024-08-17T17:37:29.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:29.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:29.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:29.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:29.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:29.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:29.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:29.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:29.678 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:29.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:29.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:29.902 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:37:29.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:29.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:30.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:30.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:29 smithi175 ceph-mon[101434]: 8.3 scrub starts 2024-08-17T17:37:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:29 smithi175 ceph-mon[101434]: pgmap v985: 99 pgs: 1 peering, 1 active+clean+scrubbing+deep, 1 unknown, 96 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-08-17T17:37:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3636678973' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:29 smithi175 ceph-mon[101434]: 8.3 scrub ok 2024-08-17T17:37:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[104516]: 8.3 scrub starts 2024-08-17T17:37:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[104516]: pgmap v985: 99 pgs: 1 peering, 1 active+clean+scrubbing+deep, 1 unknown, 96 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-08-17T17:37:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3636678973' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[104516]: 8.3 scrub ok 2024-08-17T17:37:30.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[109246]: 8.3 scrub starts 2024-08-17T17:37:30.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[109246]: pgmap v985: 99 pgs: 1 peering, 1 active+clean+scrubbing+deep, 1 unknown, 96 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2024-08-17T17:37:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3636678973' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:29 smithi064 ceph-mon[109246]: 8.3 scrub ok 2024-08-17T17:37:30.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.457 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:30.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:30.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:30.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:30.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:30.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:30.893 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:30.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:31.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:31.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:30 smithi175 ceph-mon[101434]: 8.11 scrub starts 2024-08-17T17:37:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:30 smithi175 ceph-mon[101434]: 8.4 scrub starts 2024-08-17T17:37:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:30 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:30 smithi175 ceph-mon[101434]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[104516]: 8.11 scrub starts 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[104516]: 8.4 scrub starts 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[104516]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[109246]: 8.11 scrub starts 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[109246]: 8.4 scrub starts 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:31.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:30 smithi064 ceph-mon[109246]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:37:31.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:31.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:31.293 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:31.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:31.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.602 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:31.868 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:31.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:31.994 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:32.096 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.120 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:32.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:31 smithi175 ceph-mon[101434]: 8.13 scrub starts 2024-08-17T17:37:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:31 smithi175 ceph-mon[101434]: pgmap v987: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:37:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[104516]: 8.13 scrub starts 2024-08-17T17:37:32.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[104516]: pgmap v987: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:32.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:37:32.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[109246]: 8.13 scrub starts 2024-08-17T17:37:32.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[109246]: pgmap v987: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:37:32.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:37:32.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:32.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:32.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:32.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:32.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:32.526 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:32.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:32.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:32.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:32.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:32.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:32.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:37:32.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:32.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:33.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:33.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:33.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:32 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:37:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:32 smithi175 ceph-mon[101434]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:37:33.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:37:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:32 smithi064 ceph-mon[104516]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:37:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:37:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:32 smithi064 ceph-mon[109246]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:37:33.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.296 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.365 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:37:33.365 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.1.0:127211 8.1 e293/289 8:844e0025:::benchmark_data_smithi064_242541_object87293:head v 293'11804, mlcod=293'11802)", 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:33.362592+0000", 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0020104089999999999, 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020303119999999998, 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362592+0000", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362592+0000", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362593+0000", 2024-08-17T17:37:33.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4900000000000004e-07 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362600+0000", 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0690000000000004e-06 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362614+0000", 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4285e-05 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362622+0000", 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3669999999999997e-06 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362683+0000", 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0328000000000002e-05 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:37:33.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:33.362703+0000", 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0122000000000001e-05 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2024-08-17T17:37:33.370 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:37:33.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:33.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:33.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.665 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:33.766 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:33.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:33.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:33.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:33.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:34.010 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:33] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-08-17T17:37:34.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: 8.11 scrub starts 2024-08-17T17:37:34.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:37:34.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: pgmap v990: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:34.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[104516]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: 8.11 scrub starts 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: pgmap v990: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:37:34.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:33 smithi064 ceph-mon[109246]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:37:34.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:34.049 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:34.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:34.176 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:34.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:34.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: 8.11 scrub starts 2024-08-17T17:37:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:37:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: pgmap v990: 99 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 96 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:37:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:37:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:33 smithi175 ceph-mon[101434]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:37:34.431 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:34.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:37:34.679 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:34.683 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.721 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:37:34.722 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:34.749 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:37:34.750 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:34.778 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:37:34.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:34.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:34.808 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:37:34.808 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:34.820 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:34.835 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:37:34.835 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:34.857 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:37:34.857 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:34.881 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:37:34.881 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:34.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:34.945 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:34.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:35.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:35.075 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:35.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[104516]: 8.13 scrub starts 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[104516]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[104516]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2749811030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[104516]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[109246]: 8.13 scrub starts 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[109246]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[109246]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2749811030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:34 smithi064 ceph-mon[109246]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:37:35.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:35.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:35.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:37:35.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:34 smithi175 ceph-mon[101434]: 8.13 scrub starts 2024-08-17T17:37:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:34 smithi175 ceph-mon[101434]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:34 smithi175 ceph-mon[101434]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 291'11322 2024-08-17T17:37:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2749811030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:35.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:34 smithi175 ceph-mon[101434]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:37:35.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:35.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:35.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:35.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:35.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:35.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:35.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:35.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:36.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:36.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:36.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[104516]: 8.13 scrub starts 2024-08-17T17:37:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[104516]: pgmap v993: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 2 active+clean+scrubbing+deep, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[104516]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:37:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[109246]: 8.13 scrub starts 2024-08-17T17:37:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[109246]: pgmap v993: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 2 active+clean+scrubbing+deep, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:35 smithi064 ceph-mon[109246]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:37:36.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:36.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:36.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:36.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:35 smithi175 ceph-mon[101434]: 8.13 scrub starts 2024-08-17T17:37:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:35 smithi175 ceph-mon[101434]: pgmap v993: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 2 active+clean+scrubbing+deep, 95 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-08-17T17:37:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:35 smithi175 ceph-mon[101434]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:37:36.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:36.544 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:36.566 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:36.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:36.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:36.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:36.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:36.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:36.926 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.957 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:36.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:36.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[104516]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[104516]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[104516]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[109246]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[109246]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:36 smithi064 ceph-mon[109246]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:36.984 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:37:36.987 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:37:37.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:37.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:37.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:37.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:37.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:37.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:37.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:36 smithi175 ceph-mon[101434]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:36 smithi175 ceph-mon[101434]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:37.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:36 smithi175 ceph-mon[101434]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 292'10541 2024-08-17T17:37:37.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:37.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:37.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:37.612 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:37.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:37.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:37.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:37.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:37.912 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:38.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.054 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:38.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:37 smithi064 ceph-mon[104516]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:38.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:37 smithi064 ceph-mon[104516]: pgmap v995: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 3626/222000 objects misplaced (1.633%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:37:38.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:37 smithi064 ceph-mon[109246]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:38.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:37 smithi064 ceph-mon[109246]: pgmap v995: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 3626/222000 objects misplaced (1.633%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:37:38.174 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:38.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.212 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:38.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:38.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:38.349 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:38.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:38.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:37 smithi175 ceph-mon[101434]: from='client.35628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:38.414 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:37 smithi175 ceph-mon[101434]: pgmap v995: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 3626/222000 objects misplaced (1.633%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:37:38.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:38.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.538 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":994,"stamp":"2024-08-17T17:37:35.903641+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4861597693,"num_objects":74139,"num_object_clones":0,"num_object_copies":222417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3626,"num_objects_unfound":0,"num_objects_dirty":74139,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":135652,"num_write_kb":6149922,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":548,"num_bytes_recovered":38215491,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":232065,"ondisk_log_size":232065,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18526680,"kb_used_data":14394024,"kb_used_omap":46,"kb_used_meta":4132497,"kb_avail":731434536,"statfs":{"total":767960285184,"available":748988964864,"internally_reserved":0,"allocated":14739480576,"data_stored":14727127070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":4231677171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,5,19,10,24,23,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-130482176,"num_objects":-1991,"num_object_clones":0,"num_object_copies":-5973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3626,"num_objects_unfound":0,"num_objects_dirty":-1991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1991,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":428,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.008994"},"pg_stats":[{"pgid":"8.18","version":"294'10588","reported_seq":10629,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.782065+0000","last_change":"2024-08-17T17:37:24.939954+0000","last_active":"2024-08-17T17:37:34.782065+0000","last_peered":"2024-08-17T17:37:34.782065+0000","last_clean":"2024-08-17T17:37:34.782065+0000","last_became_active":"2024-08-17T17:37:24.922914+0000","last_became_peered":"2024-08-17T17:37:24.922914+0000","last_unstale":"2024-08-17T17:37:34.782065+0000","last_undegraded":"2024-08-17T17:37:34.782065+0000","last_fullsized":"2024-08-17T17:37:34.782065+0000","mapping_epoch":289,"log_start":"278'400","ondisk_log_start":"278'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10188,"log_dups_size":0,"ondisk_log_size":10188,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151486464,"num_objects":2312,"num_object_clones":0,"num_object_copies":6936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5387,"num_write_kb":246336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":446,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.902357+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:37:33.902357+0000","last_peered":"2024-08-17T17:37:33.902357+0000","last_clean":"2024-08-17T17:37:33.902357+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:37:33.902357+0000","last_undegraded":"2024-08-17T17:37:33.902357+0000","last_fullsized":"2024-08-17T17:37:33.902357+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":289,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.855705+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:37:24.855705+0000","last_peered":"2024-08-17T17:37:24.855705+0000","last_clean":"2024-08-17T17:37:24.855705+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:37:24.855705+0000","last_undegraded":"2024-08-17T17:37:24.855705+0000","last_fullsized":"2024-08-17T17:37:24.855705+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"294'10362","reported_seq":10403,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.742482+0000","last_change":"2024-08-17T17:37:24.937674+0000","last_active":"2024-08-17T17:37:34.742482+0000","last_peered":"2024-08-17T17:37:34.742482+0000","last_clean":"2024-08-17T17:37:34.742482+0000","last_became_active":"2024-08-17T17:37:24.919618+0000","last_became_peered":"2024-08-17T17:37:24.919618+0000","last_unstale":"2024-08-17T17:37:34.742482+0000","last_undegraded":"2024-08-17T17:37:34.742482+0000","last_fullsized":"2024-08-17T17:37:34.742482+0000","mapping_epoch":289,"log_start":"278'100","ondisk_log_start":"278'100","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10262,"log_dups_size":0,"ondisk_log_size":10262,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151355392,"num_objects":2310,"num_object_clones":0,"num_object_copies":6930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5279,"num_write_kb":242816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896607+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:37:33.896607+0000","last_peered":"2024-08-17T17:37:33.896607+0000","last_clean":"2024-08-17T17:37:33.896607+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:37:33.896607+0000","last_undegraded":"2024-08-17T17:37:33.896607+0000","last_fullsized":"2024-08-17T17:37:33.896607+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":440,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900427+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:37:34.900427+0000","last_peered":"2024-08-17T17:37:34.900427+0000","last_clean":"2024-08-17T17:37:34.900427+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:37:34.900427+0000","last_undegraded":"2024-08-17T17:37:34.900427+0000","last_fullsized":"2024-08-17T17:37:34.900427+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":440,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901659+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:37:34.901659+0000","last_peered":"2024-08-17T17:37:34.901659+0000","last_clean":"2024-08-17T17:37:34.901659+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:37:34.901659+0000","last_undegraded":"2024-08-17T17:37:34.901659+0000","last_fullsized":"2024-08-17T17:37:34.901659+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.855411+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:37:24.855411+0000","last_peered":"2024-08-17T17:37:24.855411+0000","last_clean":"2024-08-17T17:37:24.855411+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:37:24.855411+0000","last_undegraded":"2024-08-17T17:37:24.855411+0000","last_fullsized":"2024-08-17T17:37:24.855411+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":446,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.902664+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:37:33.902664+0000","last_peered":"2024-08-17T17:37:33.902664+0000","last_clean":"2024-08-17T17:37:33.902664+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:37:33.902664+0000","last_undegraded":"2024-08-17T17:37:33.902664+0000","last_fullsized":"2024-08-17T17:37:33.902664+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":226,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-17T17:37:32.904013+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:37:32.904013+0000","last_peered":"2024-08-17T17:37:32.904013+0000","last_clean":"2024-08-17T17:37:32.904013+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:37:32.904013+0000","last_undegraded":"2024-08-17T17:37:32.904013+0000","last_fullsized":"2024-08-17T17:37:32.904013+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"294'10521","reported_seq":10573,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.815623+0000","last_change":"2024-08-17T17:37:30.053215+0000","last_active":"2024-08-17T17:37:34.815623+0000","last_peered":"2024-08-17T17:37:34.815623+0000","last_clean":"2024-08-17T17:37:34.815623+0000","last_became_active":"2024-08-17T17:37:24.862536+0000","last_became_peered":"2024-08-17T17:37:24.862536+0000","last_unstale":"2024-08-17T17:37:34.815623+0000","last_undegraded":"2024-08-17T17:37:34.815623+0000","last_fullsized":"2024-08-17T17:37:34.815623+0000","mapping_epoch":274,"log_start":"278'300","ondisk_log_start":"278'300","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":499,"log_size":10221,"log_dups_size":0,"ondisk_log_size":10221,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150831104,"num_objects":2301,"num_object_clones":0,"num_object_copies":6903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5356,"num_write_kb":245056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856268+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:37:24.856268+0000","last_peered":"2024-08-17T17:37:24.856268+0000","last_clean":"2024-08-17T17:37:24.856268+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:37:24.856268+0000","last_undegraded":"2024-08-17T17:37:24.856268+0000","last_fullsized":"2024-08-17T17:37:24.856268+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":87,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896375+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:37:33.896375+0000","last_peered":"2024-08-17T17:37:33.896375+0000","last_clean":"2024-08-17T17:37:33.896375+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:37:33.896375+0000","last_undegraded":"2024-08-17T17:37:33.896375+0000","last_fullsized":"2024-08-17T17:37:33.896375+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"294'10733","reported_seq":10772,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.508902+0000","last_change":"2024-08-17T17:37:24.880391+0000","last_active":"2024-08-17T17:37:34.508902+0000","last_peered":"2024-08-17T17:37:34.508902+0000","last_clean":"2024-08-17T17:37:34.508902+0000","last_became_active":"2024-08-17T17:37:24.862433+0000","last_became_peered":"2024-08-17T17:37:24.862433+0000","last_unstale":"2024-08-17T17:37:34.508902+0000","last_undegraded":"2024-08-17T17:37:34.508902+0000","last_fullsized":"2024-08-17T17:37:34.508902+0000","mapping_epoch":274,"log_start":"278'1100","ondisk_log_start":"278'1100","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9633,"log_dups_size":0,"ondisk_log_size":9633,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153124864,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5461,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":286,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856580+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:37:24.856580+0000","last_peered":"2024-08-17T17:37:24.856580+0000","last_clean":"2024-08-17T17:37:24.856580+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:37:24.856580+0000","last_undegraded":"2024-08-17T17:37:24.856580+0000","last_fullsized":"2024-08-17T17:37:24.856580+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":181,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.896201+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:37:29.896201+0000","last_peered":"2024-08-17T17:37:29.896201+0000","last_clean":"2024-08-17T17:37:29.896201+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:37:29.896201+0000","last_undegraded":"2024-08-17T17:37:29.896201+0000","last_fullsized":"2024-08-17T17:37:29.896201+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"294'10361","reported_seq":10457,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901051+0000","last_change":"2024-08-17T17:37:26.287987+0000","last_active":"2024-08-17T17:37:34.901051+0000","last_peered":"2024-08-17T17:37:34.901051+0000","last_clean":"2024-08-17T17:37:34.901051+0000","last_became_active":"2024-08-17T17:37:24.868093+0000","last_became_peered":"2024-08-17T17:37:24.868093+0000","last_unstale":"2024-08-17T17:37:34.901051+0000","last_undegraded":"2024-08-17T17:37:34.901051+0000","last_fullsized":"2024-08-17T17:37:34.901051+0000","mapping_epoch":274,"log_start":"278'800","ondisk_log_start":"278'800","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10217","last_scrub_stamp":"2024-08-17T17:37:26.287558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:26.287558+0000","objects_scrubbed":2490,"log_size":9561,"log_dups_size":0,"ondisk_log_size":9561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:23.491632+0000","scrub_duration":708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152961024,"num_objects":2334,"num_object_clones":0,"num_object_copies":7002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":156,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":303,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.857887+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:37:24.857887+0000","last_peered":"2024-08-17T17:37:24.857887+0000","last_clean":"2024-08-17T17:37:24.857887+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:37:24.857887+0000","last_undegraded":"2024-08-17T17:37:24.857887+0000","last_fullsized":"2024-08-17T17:37:24.857887+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":206,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-17T17:37:32.904207+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:37:32.904207+0000","last_peered":"2024-08-17T17:37:32.904207+0000","last_clean":"2024-08-17T17:37:32.904207+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:37:32.904207+0000","last_undegraded":"2024-08-17T17:37:32.904207+0000","last_fullsized":"2024-08-17T17:37:32.904207+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"294'10509","reported_seq":10606,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.904020+0000","last_change":"2024-08-17T17:37:26.443446+0000","last_active":"2024-08-17T17:37:34.904020+0000","last_peered":"2024-08-17T17:37:34.904020+0000","last_clean":"2024-08-17T17:37:34.904020+0000","last_became_active":"2024-08-17T17:37:24.890776+0000","last_became_peered":"2024-08-17T17:37:24.890776+0000","last_unstale":"2024-08-17T17:37:34.904020+0000","last_undegraded":"2024-08-17T17:37:34.904020+0000","last_fullsized":"2024-08-17T17:37:34.904020+0000","mapping_epoch":274,"log_start":"279'2600","ondisk_log_start":"279'2600","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10372","last_scrub_stamp":"2024-08-17T17:37:26.443046+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:26.443046+0000","objects_scrubbed":2396,"log_size":7909,"log_dups_size":0,"ondisk_log_size":7909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:55.937652+0000","scrub_duration":987,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":148,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896652+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:37:33.896652+0000","last_peered":"2024-08-17T17:37:33.896652+0000","last_clean":"2024-08-17T17:37:33.896652+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:37:33.896652+0000","last_undegraded":"2024-08-17T17:37:33.896652+0000","last_fullsized":"2024-08-17T17:37:33.896652+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":455,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900731+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:37:34.900731+0000","last_peered":"2024-08-17T17:37:34.900731+0000","last_clean":"2024-08-17T17:37:34.900731+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:37:34.900731+0000","last_undegraded":"2024-08-17T17:37:34.900731+0000","last_fullsized":"2024-08-17T17:37:34.900731+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"292'10366","reported_seq":10460,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.900681+0000","last_change":"2024-08-17T17:37:25.992621+0000","last_active":"2024-08-17T17:37:33.900681+0000","last_peered":"2024-08-17T17:37:33.900681+0000","last_clean":"2024-08-17T17:37:33.900681+0000","last_became_active":"2024-08-17T17:37:24.922665+0000","last_became_peered":"2024-08-17T17:37:24.922665+0000","last_unstale":"2024-08-17T17:37:33.900681+0000","last_undegraded":"2024-08-17T17:37:33.900681+0000","last_fullsized":"2024-08-17T17:37:33.900681+0000","mapping_epoch":274,"log_start":"278'2000","ondisk_log_start":"278'2000","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10237","last_scrub_stamp":"2024-08-17T17:37:25.992076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:25.992076+0000","objects_scrubbed":2320,"log_size":8366,"log_dups_size":0,"ondisk_log_size":8366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:20.475523+0000","scrub_duration":812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142475264,"num_objects":2174,"num_object_clones":0,"num_object_copies":6522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":146,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":468,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.902151+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:37:33.902151+0000","last_peered":"2024-08-17T17:37:33.902151+0000","last_clean":"2024-08-17T17:37:33.902151+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:37:33.902151+0000","last_undegraded":"2024-08-17T17:37:33.902151+0000","last_fullsized":"2024-08-17T17:37:33.902151+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":195,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.858923+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:37:24.858923+0000","last_peered":"2024-08-17T17:37:24.858923+0000","last_clean":"2024-08-17T17:37:24.858923+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:37:24.858923+0000","last_undegraded":"2024-08-17T17:37:24.858923+0000","last_fullsized":"2024-08-17T17:37:24.858923+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"295'12468","reported_seq":12590,"reported_epoch":295,"state":"active+clean+scrubbing+deep","last_fresh":"2024-08-17T17:37:35.524248+0000","last_change":"2024-08-17T17:37:26.489586+0000","last_active":"2024-08-17T17:37:35.524248+0000","last_peered":"2024-08-17T17:37:35.524248+0000","last_clean":"2024-08-17T17:37:35.524248+0000","last_became_active":"2024-08-17T17:37:24.906036+0000","last_became_peered":"2024-08-17T17:37:24.906036+0000","last_unstale":"2024-08-17T17:37:35.524248+0000","last_undegraded":"2024-08-17T17:37:35.524248+0000","last_fullsized":"2024-08-17T17:37:35.524248+0000","mapping_epoch":274,"log_start":"279'2600","ondisk_log_start":"279'2600","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":1949,"log_size":9868,"log_dups_size":0,"ondisk_log_size":9868,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 9s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151977984,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5347,"num_write_kb":245312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":210,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.863333+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:37:24.863333+0000","last_peered":"2024-08-17T17:37:24.863333+0000","last_clean":"2024-08-17T17:37:24.863333+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:37:24.863333+0000","last_undegraded":"2024-08-17T17:37:24.863333+0000","last_fullsized":"2024-08-17T17:37:24.863333+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.857443+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:37:24.857443+0000","last_peered":"2024-08-17T17:37:24.857443+0000","last_clean":"2024-08-17T17:37:24.857443+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:37:24.857443+0000","last_undegraded":"2024-08-17T17:37:24.857443+0000","last_fullsized":"2024-08-17T17:37:24.857443+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"291'10500","reported_seq":10600,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:31.791450+0000","last_change":"2024-08-17T17:37:27.632902+0000","last_active":"2024-08-17T17:37:31.791450+0000","last_peered":"2024-08-17T17:37:31.791450+0000","last_clean":"2024-08-17T17:37:31.791450+0000","last_became_active":"2024-08-17T17:37:24.892206+0000","last_became_peered":"2024-08-17T17:37:24.892206+0000","last_unstale":"2024-08-17T17:37:31.791450+0000","last_undegraded":"2024-08-17T17:37:31.791450+0000","last_fullsized":"2024-08-17T17:37:31.791450+0000","mapping_epoch":274,"log_start":"279'2200","ondisk_log_start":"279'2200","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10412","last_scrub_stamp":"2024-08-17T17:37:27.632741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:27.632741+0000","objects_scrubbed":2417,"log_size":8300,"log_dups_size":0,"ondisk_log_size":8300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:59.051240+0000","scrub_duration":0.43553773699999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":181,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.896035+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:37:29.896035+0000","last_peered":"2024-08-17T17:37:29.896035+0000","last_clean":"2024-08-17T17:37:29.896035+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:37:29.896035+0000","last_undegraded":"2024-08-17T17:37:29.896035+0000","last_fullsized":"2024-08-17T17:37:29.896035+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":195,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856870+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:37:24.856870+0000","last_peered":"2024-08-17T17:37:24.856870+0000","last_clean":"2024-08-17T17:37:24.856870+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:37:24.856870+0000","last_undegraded":"2024-08-17T17:37:24.856870+0000","last_fullsized":"2024-08-17T17:37:24.856870+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"294'10288","reported_seq":10386,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903798+0000","last_change":"2024-08-17T17:37:29.268709+0000","last_active":"2024-08-17T17:37:34.903798+0000","last_peered":"2024-08-17T17:37:34.903798+0000","last_clean":"2024-08-17T17:37:34.903798+0000","last_became_active":"2024-08-17T17:37:24.929016+0000","last_became_peered":"2024-08-17T17:37:24.929016+0000","last_unstale":"2024-08-17T17:37:34.903798+0000","last_undegraded":"2024-08-17T17:37:34.903798+0000","last_fullsized":"2024-08-17T17:37:34.903798+0000","mapping_epoch":274,"log_start":"279'2400","ondisk_log_start":"279'2400","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10197","last_scrub_stamp":"2024-08-17T17:37:29.268581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:29.268581+0000","objects_scrubbed":2413,"log_size":7888,"log_dups_size":0,"ondisk_log_size":7888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:53.647624+0000","scrub_duration":753,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":448,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.902047+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:37:33.902047+0000","last_peered":"2024-08-17T17:37:33.902047+0000","last_clean":"2024-08-17T17:37:33.902047+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:37:33.902047+0000","last_undegraded":"2024-08-17T17:37:33.902047+0000","last_fullsized":"2024-08-17T17:37:33.902047+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":430,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.902295+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:37:29.902295+0000","last_peered":"2024-08-17T17:37:29.902295+0000","last_clean":"2024-08-17T17:37:29.902295+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:37:29.902295+0000","last_undegraded":"2024-08-17T17:37:29.902295+0000","last_fullsized":"2024-08-17T17:37:29.902295+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"294'10351","reported_seq":10392,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.657224+0000","last_change":"2024-08-17T17:37:24.919277+0000","last_active":"2024-08-17T17:37:34.657224+0000","last_peered":"2024-08-17T17:37:34.657224+0000","last_clean":"2024-08-17T17:37:34.657224+0000","last_became_active":"2024-08-17T17:37:24.907010+0000","last_became_peered":"2024-08-17T17:37:24.907010+0000","last_unstale":"2024-08-17T17:37:34.657224+0000","last_undegraded":"2024-08-17T17:37:34.657224+0000","last_fullsized":"2024-08-17T17:37:34.657224+0000","mapping_epoch":274,"log_start":"278'100","ondisk_log_start":"278'100","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10251,"log_dups_size":0,"ondisk_log_size":10251,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152076288,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5267,"num_write_kb":242816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":605,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856480+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:37:24.856480+0000","last_peered":"2024-08-17T17:37:24.856480+0000","last_clean":"2024-08-17T17:37:24.856480+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:37:24.856480+0000","last_undegraded":"2024-08-17T17:37:24.856480+0000","last_fullsized":"2024-08-17T17:37:24.856480+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":886,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896423+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:37:33.896423+0000","last_peered":"2024-08-17T17:37:33.896423+0000","last_clean":"2024-08-17T17:37:33.896423+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:37:33.896423+0000","last_undegraded":"2024-08-17T17:37:33.896423+0000","last_fullsized":"2024-08-17T17:37:33.896423+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":431,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901188+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:37:33.901188+0000","last_peered":"2024-08-17T17:37:33.901188+0000","last_clean":"2024-08-17T17:37:33.901188+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:37:33.901188+0000","last_undegraded":"2024-08-17T17:37:33.901188+0000","last_fullsized":"2024-08-17T17:37:33.901188+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":194,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.857271+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:37:24.857271+0000","last_peered":"2024-08-17T17:37:24.857271+0000","last_clean":"2024-08-17T17:37:24.857271+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:37:24.857271+0000","last_undegraded":"2024-08-17T17:37:24.857271+0000","last_fullsized":"2024-08-17T17:37:24.857271+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"294'10708","reported_seq":10766,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903175+0000","last_change":"2024-08-17T17:34:48.462773+0000","last_active":"2024-08-17T17:37:34.903175+0000","last_peered":"2024-08-17T17:37:34.903175+0000","last_clean":"2024-08-17T17:37:34.903175+0000","last_became_active":"2024-08-17T17:34:48.462475+0000","last_became_peered":"2024-08-17T17:34:48.462475+0000","last_unstale":"2024-08-17T17:37:34.903175+0000","last_undegraded":"2024-08-17T17:37:34.903175+0000","last_fullsized":"2024-08-17T17:37:34.903175+0000","mapping_epoch":274,"log_start":"280'3726","ondisk_log_start":"280'3726","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:33.030764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10708,"num_write_kb":491264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":668,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856388+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:37:24.856388+0000","last_peered":"2024-08-17T17:37:24.856388+0000","last_clean":"2024-08-17T17:37:24.856388+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:37:24.856388+0000","last_undegraded":"2024-08-17T17:37:24.856388+0000","last_fullsized":"2024-08-17T17:37:24.856388+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":455,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.906748+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:37:34.906748+0000","last_peered":"2024-08-17T17:37:34.906748+0000","last_clean":"2024-08-17T17:37:34.906748+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:37:34.906748+0000","last_undegraded":"2024-08-17T17:37:34.906748+0000","last_fullsized":"2024-08-17T17:37:34.906748+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":452,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901756+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:37:34.901756+0000","last_peered":"2024-08-17T17:37:34.901756+0000","last_clean":"2024-08-17T17:37:34.901756+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:37:34.901756+0000","last_undegraded":"2024-08-17T17:37:34.901756+0000","last_fullsized":"2024-08-17T17:37:34.901756+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"293'10763","reported_seq":10819,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901979+0000","last_change":"2024-08-17T17:34:48.457912+0000","last_active":"2024-08-17T17:37:33.901979+0000","last_peered":"2024-08-17T17:37:33.901979+0000","last_clean":"2024-08-17T17:37:33.901979+0000","last_became_active":"2024-08-17T17:34:48.457628+0000","last_became_peered":"2024-08-17T17:34:48.457628+0000","last_unstale":"2024-08-17T17:37:33.901979+0000","last_undegraded":"2024-08-17T17:37:33.901979+0000","last_fullsized":"2024-08-17T17:37:33.901979+0000","mapping_epoch":274,"log_start":"279'3235","ondisk_log_start":"279'3235","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:18.747117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311754752,"num_objects":4757,"num_object_clones":0,"num_object_copies":14271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10763,"num_write_kb":496640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":845,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.900986+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:37:29.900986+0000","last_peered":"2024-08-17T17:37:29.900986+0000","last_clean":"2024-08-17T17:37:29.900986+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:37:29.900986+0000","last_undegraded":"2024-08-17T17:37:29.900986+0000","last_fullsized":"2024-08-17T17:37:29.900986+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":444,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.904896+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:37:34.904896+0000","last_peered":"2024-08-17T17:37:34.904896+0000","last_clean":"2024-08-17T17:37:34.904896+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:37:34.904896+0000","last_undegraded":"2024-08-17T17:37:34.904896+0000","last_fullsized":"2024-08-17T17:37:34.904896+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":492,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.904443+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:37:34.904443+0000","last_peered":"2024-08-17T17:37:34.904443+0000","last_clean":"2024-08-17T17:37:34.904443+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:37:34.904443+0000","last_undegraded":"2024-08-17T17:37:34.904443+0000","last_fullsized":"2024-08-17T17:37:34.904443+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"294'10601","reported_seq":10642,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.747034+0000","last_change":"2024-08-17T17:37:24.918681+0000","last_active":"2024-08-17T17:37:34.747034+0000","last_peered":"2024-08-17T17:37:34.747034+0000","last_clean":"2024-08-17T17:37:34.747034+0000","last_became_active":"2024-08-17T17:37:24.885665+0000","last_became_peered":"2024-08-17T17:37:24.885665+0000","last_unstale":"2024-08-17T17:37:34.747034+0000","last_undegraded":"2024-08-17T17:37:34.747034+0000","last_fullsized":"2024-08-17T17:37:34.747034+0000","mapping_epoch":274,"log_start":"278'400","ondisk_log_start":"278'400","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10201,"log_dups_size":0,"ondisk_log_size":10201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150634496,"num_objects":2298,"num_object_clones":0,"num_object_copies":6894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":246336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1009,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896622+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:37:33.896622+0000","last_peered":"2024-08-17T17:37:33.896622+0000","last_clean":"2024-08-17T17:37:33.896622+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:37:33.896622+0000","last_undegraded":"2024-08-17T17:37:33.896622+0000","last_fullsized":"2024-08-17T17:37:33.896622+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":262,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.900986+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:37:29.900986+0000","last_peered":"2024-08-17T17:37:29.900986+0000","last_clean":"2024-08-17T17:37:29.900986+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:37:29.900986+0000","last_undegraded":"2024-08-17T17:37:29.900986+0000","last_fullsized":"2024-08-17T17:37:29.900986+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903650+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:37:34.903650+0000","last_peered":"2024-08-17T17:37:34.903650+0000","last_clean":"2024-08-17T17:37:34.903650+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:37:34.903650+0000","last_undegraded":"2024-08-17T17:37:34.903650+0000","last_fullsized":"2024-08-17T17:37:34.903650+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"294'10600","reported_seq":10658,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900964+0000","last_change":"2024-08-17T17:34:48.464724+0000","last_active":"2024-08-17T17:37:34.900964+0000","last_peered":"2024-08-17T17:37:34.900964+0000","last_clean":"2024-08-17T17:37:34.900964+0000","last_became_active":"2024-08-17T17:34:48.464408+0000","last_became_peered":"2024-08-17T17:34:48.464408+0000","last_unstale":"2024-08-17T17:37:34.900964+0000","last_undegraded":"2024-08-17T17:37:34.900964+0000","last_fullsized":"2024-08-17T17:37:34.900964+0000","mapping_epoch":274,"log_start":"279'2214","ondisk_log_start":"279'2214","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8386,"log_dups_size":0,"ondisk_log_size":8386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:42.029003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303562752,"num_objects":4632,"num_object_clones":0,"num_object_copies":13896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10600,"num_write_kb":487424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":78,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896542+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:37:33.896542+0000","last_peered":"2024-08-17T17:37:33.896542+0000","last_clean":"2024-08-17T17:37:33.896542+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:37:33.896542+0000","last_undegraded":"2024-08-17T17:37:33.896542+0000","last_fullsized":"2024-08-17T17:37:33.896542+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":737,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903842+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:37:34.903842+0000","last_peered":"2024-08-17T17:37:34.903842+0000","last_clean":"2024-08-17T17:37:34.903842+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:37:34.903842+0000","last_undegraded":"2024-08-17T17:37:34.903842+0000","last_fullsized":"2024-08-17T17:37:34.903842+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":461,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903489+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:37:34.903489+0000","last_peered":"2024-08-17T17:37:34.903489+0000","last_clean":"2024-08-17T17:37:34.903489+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:37:34.903489+0000","last_undegraded":"2024-08-17T17:37:34.903489+0000","last_fullsized":"2024-08-17T17:37:34.903489+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"293'10763","reported_seq":10819,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.902797+0000","last_change":"2024-08-17T17:34:48.463247+0000","last_active":"2024-08-17T17:37:33.902797+0000","last_peered":"2024-08-17T17:37:33.902797+0000","last_clean":"2024-08-17T17:37:33.902797+0000","last_became_active":"2024-08-17T17:34:48.462966+0000","last_became_peered":"2024-08-17T17:34:48.462966+0000","last_unstale":"2024-08-17T17:37:33.902797+0000","last_undegraded":"2024-08-17T17:37:33.902797+0000","last_fullsized":"2024-08-17T17:37:33.902797+0000","mapping_epoch":274,"log_start":"279'3223","ondisk_log_start":"279'3223","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:14.540272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307298304,"num_objects":4689,"num_object_clones":0,"num_object_copies":14067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10763,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":458,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903743+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:37:34.903743+0000","last_peered":"2024-08-17T17:37:34.903743+0000","last_clean":"2024-08-17T17:37:34.903743+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:37:34.903743+0000","last_undegraded":"2024-08-17T17:37:34.903743+0000","last_fullsized":"2024-08-17T17:37:34.903743+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":857,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.857897+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:37:24.857897+0000","last_peered":"2024-08-17T17:37:24.857897+0000","last_clean":"2024-08-17T17:37:24.857897+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:37:24.857897+0000","last_undegraded":"2024-08-17T17:37:24.857897+0000","last_fullsized":"2024-08-17T17:37:24.857897+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":455,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900624+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:37:34.900624+0000","last_peered":"2024-08-17T17:37:34.900624+0000","last_clean":"2024-08-17T17:37:34.900624+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:37:34.900624+0000","last_undegraded":"2024-08-17T17:37:34.900624+0000","last_fullsized":"2024-08-17T17:37:34.900624+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"294'10660","reported_seq":10718,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900502+0000","last_change":"2024-08-17T17:34:48.462499+0000","last_active":"2024-08-17T17:37:34.900502+0000","last_peered":"2024-08-17T17:37:34.900502+0000","last_clean":"2024-08-17T17:37:34.900502+0000","last_became_active":"2024-08-17T17:34:48.461996+0000","last_became_peered":"2024-08-17T17:34:48.461996+0000","last_unstale":"2024-08-17T17:37:34.900502+0000","last_undegraded":"2024-08-17T17:37:34.900502+0000","last_fullsized":"2024-08-17T17:37:34.900502+0000","mapping_epoch":274,"log_start":"279'2284","ondisk_log_start":"279'2284","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8376,"log_dups_size":0,"ondisk_log_size":8376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:08.325353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304218112,"num_objects":4642,"num_object_clones":0,"num_object_copies":13926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10660,"num_write_kb":489664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":446,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.903027+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:37:33.903027+0000","last_peered":"2024-08-17T17:37:33.903027+0000","last_clean":"2024-08-17T17:37:33.903027+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:37:33.903027+0000","last_undegraded":"2024-08-17T17:37:33.903027+0000","last_fullsized":"2024-08-17T17:37:33.903027+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":968,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.904321+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:37:34.904321+0000","last_peered":"2024-08-17T17:37:34.904321+0000","last_clean":"2024-08-17T17:37:34.904321+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:37:34.904321+0000","last_undegraded":"2024-08-17T17:37:34.904321+0000","last_fullsized":"2024-08-17T17:37:34.904321+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":444,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901693+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:37:33.901693+0000","last_peered":"2024-08-17T17:37:33.901693+0000","last_clean":"2024-08-17T17:37:33.901693+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:37:33.901693+0000","last_undegraded":"2024-08-17T17:37:33.901693+0000","last_fullsized":"2024-08-17T17:37:33.901693+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"294'10604","reported_seq":10636,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.758004+0000","last_change":"2024-08-17T17:34:48.463451+0000","last_active":"2024-08-17T17:37:34.758004+0000","last_peered":"2024-08-17T17:37:34.758004+0000","last_clean":"2024-08-17T17:37:34.758004+0000","last_became_active":"2024-08-17T17:34:48.463124+0000","last_became_peered":"2024-08-17T17:34:48.463124+0000","last_unstale":"2024-08-17T17:37:34.758004+0000","last_undegraded":"2024-08-17T17:37:34.758004+0000","last_fullsized":"2024-08-17T17:37:34.758004+0000","mapping_epoch":274,"log_start":"278'2003","ondisk_log_start":"278'2003","created":274,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8601,"log_dups_size":0,"ondisk_log_size":8601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:58:26.982388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305201176,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10604,"num_write_kb":488321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":467,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901600+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:37:34.901600+0000","last_peered":"2024-08-17T17:37:34.901600+0000","last_clean":"2024-08-17T17:37:34.901600+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:37:34.901600+0000","last_undegraded":"2024-08-17T17:37:34.901600+0000","last_fullsized":"2024-08-17T17:37:34.901600+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":413,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856382+0000","last_change":"2024-08-17T17:31:36.157125+0000","last_active":"2024-08-17T17:37:24.856382+0000","last_peered":"2024-08-17T17:37:24.856382+0000","last_clean":"2024-08-17T17:37:24.856382+0000","last_became_active":"2024-08-17T17:31:36.156729+0000","last_became_peered":"2024-08-17T17:31:36.156729+0000","last_unstale":"2024-08-17T17:37:24.856382+0000","last_undegraded":"2024-08-17T17:37:24.856382+0000","last_fullsized":"2024-08-17T17:37:24.856382+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:18.304031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":232,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856682+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:37:24.856682+0000","last_peered":"2024-08-17T17:37:24.856682+0000","last_clean":"2024-08-17T17:37:24.856682+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:37:24.856682+0000","last_undegraded":"2024-08-17T17:37:24.856682+0000","last_fullsized":"2024-08-17T17:37:24.856682+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":195,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.857008+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:37:24.857008+0000","last_peered":"2024-08-17T17:37:24.857008+0000","last_clean":"2024-08-17T17:37:24.857008+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:37:24.857008+0000","last_undegraded":"2024-08-17T17:37:24.857008+0000","last_fullsized":"2024-08-17T17:37:24.857008+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":205,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.858563+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:37:24.858563+0000","last_peered":"2024-08-17T17:37:24.858563+0000","last_clean":"2024-08-17T17:37:24.858563+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:37:24.858563+0000","last_undegraded":"2024-08-17T17:37:24.858563+0000","last_fullsized":"2024-08-17T17:37:24.858563+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":443,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903334+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:37:34.903334+0000","last_peered":"2024-08-17T17:37:34.903334+0000","last_clean":"2024-08-17T17:37:34.903334+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:37:34.903334+0000","last_undegraded":"2024-08-17T17:37:34.903334+0000","last_fullsized":"2024-08-17T17:37:34.903334+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":290,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856930+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:37:24.856930+0000","last_peered":"2024-08-17T17:37:24.856930+0000","last_clean":"2024-08-17T17:37:24.856930+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:37:24.856930+0000","last_undegraded":"2024-08-17T17:37:24.856930+0000","last_fullsized":"2024-08-17T17:37:24.856930+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":77,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896326+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:37:33.896326+0000","last_peered":"2024-08-17T17:37:33.896326+0000","last_clean":"2024-08-17T17:37:33.896326+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:37:33.896326+0000","last_undegraded":"2024-08-17T17:37:33.896326+0000","last_fullsized":"2024-08-17T17:37:33.896326+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":409,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.900817+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:37:33.900817+0000","last_peered":"2024-08-17T17:37:33.900817+0000","last_clean":"2024-08-17T17:37:33.900817+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:37:33.900817+0000","last_undegraded":"2024-08-17T17:37:33.900817+0000","last_fullsized":"2024-08-17T17:37:33.900817+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":469,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901290+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:37:34.901290+0000","last_peered":"2024-08-17T17:37:34.901290+0000","last_clean":"2024-08-17T17:37:34.901290+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:37:34.901290+0000","last_undegraded":"2024-08-17T17:37:34.901290+0000","last_fullsized":"2024-08-17T17:37:34.901290+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":458,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901647+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:37:33.901647+0000","last_peered":"2024-08-17T17:37:33.901647+0000","last_clean":"2024-08-17T17:37:33.901647+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:37:33.901647+0000","last_undegraded":"2024-08-17T17:37:33.901647+0000","last_fullsized":"2024-08-17T17:37:33.901647+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.13","version":"294'10286","reported_seq":10362,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903620+0000","last_change":"2024-08-17T17:37:34.573406+0000","last_active":"2024-08-17T17:37:34.903620+0000","last_peered":"2024-08-17T17:37:34.903620+0000","last_clean":"2024-08-17T17:37:34.903620+0000","last_became_active":"2024-08-17T17:37:24.941057+0000","last_became_peered":"2024-08-17T17:37:24.941057+0000","last_unstale":"2024-08-17T17:37:34.903620+0000","last_undegraded":"2024-08-17T17:37:34.903620+0000","last_fullsized":"2024-08-17T17:37:34.903620+0000","mapping_epoch":289,"log_start":"279'2400","ondisk_log_start":"279'2400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7886,"log_dups_size":0,"ondisk_log_size":7886,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:37:39.540771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5236,"num_write_kb":241600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896434+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:37:33.896434+0000","last_peered":"2024-08-17T17:37:33.896434+0000","last_clean":"2024-08-17T17:37:33.896434+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:37:33.896434+0000","last_undegraded":"2024-08-17T17:37:33.896434+0000","last_fullsized":"2024-08-17T17:37:33.896434+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":462,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901773+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:37:33.901773+0000","last_peered":"2024-08-17T17:37:33.901773+0000","last_clean":"2024-08-17T17:37:33.901773+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:37:33.901773+0000","last_undegraded":"2024-08-17T17:37:33.901773+0000","last_fullsized":"2024-08-17T17:37:33.901773+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"291'10495","reported_seq":10535,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:31.827322+0000","last_change":"2024-08-17T17:37:24.948269+0000","last_active":"2024-08-17T17:37:31.827322+0000","last_peered":"2024-08-17T17:37:31.827322+0000","last_clean":"2024-08-17T17:37:31.827322+0000","last_became_active":"2024-08-17T17:37:24.927341+0000","last_became_peered":"2024-08-17T17:37:24.927341+0000","last_unstale":"2024-08-17T17:37:31.827322+0000","last_undegraded":"2024-08-17T17:37:31.827322+0000","last_fullsized":"2024-08-17T17:37:31.827322+0000","mapping_epoch":289,"log_start":"279'2200","ondisk_log_start":"279'2200","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":8295,"log_dups_size":0,"ondisk_log_size":8295,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155222016,"num_objects":2369,"num_object_clones":0,"num_object_copies":7107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5328,"num_write_kb":246272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":442,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901699+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:37:34.901699+0000","last_peered":"2024-08-17T17:37:34.901699+0000","last_clean":"2024-08-17T17:37:34.901699+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:37:34.901699+0000","last_undegraded":"2024-08-17T17:37:34.901699+0000","last_fullsized":"2024-08-17T17:37:34.901699+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":446,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903740+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:37:34.903740+0000","last_peered":"2024-08-17T17:37:34.903740+0000","last_clean":"2024-08-17T17:37:34.903740+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:37:34.903740+0000","last_undegraded":"2024-08-17T17:37:34.903740+0000","last_fullsized":"2024-08-17T17:37:34.903740+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.11","version":"292'10541","reported_seq":10614,"reported_epoch":295,"state":"remapped+peering","last_fresh":"2024-08-17T17:37:34.902587+0000","last_change":"2024-08-17T17:37:34.902587+0000","last_active":"2024-08-17T17:37:34.902587+0000","last_peered":"2024-08-17T17:37:32.626767+0000","last_clean":"2024-08-17T17:37:32.626767+0000","last_became_active":"2024-08-17T17:37:24.929119+0000","last_became_peered":"2024-08-17T17:37:24.929119+0000","last_unstale":"2024-08-17T17:37:34.902587+0000","last_undegraded":"2024-08-17T17:37:34.902587+0000","last_fullsized":"2024-08-17T17:37:34.902587+0000","mapping_epoch":295,"log_start":"279'2600","ondisk_log_start":"279'2600","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":7941,"log_dups_size":0,"ondisk_log_size":7941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150536192,"num_objects":2297,"num_object_clones":0,"num_object_copies":6891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5369,"num_write_kb":245312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":459,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901625+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:37:33.901625+0000","last_peered":"2024-08-17T17:37:33.901625+0000","last_clean":"2024-08-17T17:37:33.901625+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:37:33.901625+0000","last_undegraded":"2024-08-17T17:37:33.901625+0000","last_fullsized":"2024-08-17T17:37:33.901625+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":180,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.863498+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:37:24.863498+0000","last_peered":"2024-08-17T17:37:24.863498+0000","last_clean":"2024-08-17T17:37:24.863498+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:37:24.863498+0000","last_undegraded":"2024-08-17T17:37:24.863498+0000","last_fullsized":"2024-08-17T17:37:24.863498+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"294'11347","reported_seq":12719,"reported_epoch":295,"state":"active+remapped+backfilling","last_fresh":"2024-08-17T17:37:35.522828+0000","last_change":"2024-08-17T17:37:34.117665+0000","last_active":"2024-08-17T17:37:35.522828+0000","last_peered":"2024-08-17T17:37:35.522828+0000","last_clean":"2024-08-17T17:37:31.782438+0000","last_became_active":"2024-08-17T17:37:34.077048+0000","last_became_peered":"2024-08-17T17:37:34.077048+0000","last_unstale":"2024-08-17T17:37:35.522828+0000","last_undegraded":"2024-08-17T17:37:35.522828+0000","last_fullsized":"2024-08-17T17:37:35.522828+0000","mapping_epoch":293,"log_start":"278'2000","ondisk_log_start":"278'2000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":949,"log_size":9347,"log_dups_size":0,"ondisk_log_size":9347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146767872,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3626,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5296,"num_write_kb":241120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[1,5,3],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":2240}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":462,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-17T17:37:29.902525+0000","last_change":"2024-08-17T17:18:55.493957+0000","last_active":"2024-08-17T17:37:29.902525+0000","last_peered":"2024-08-17T17:37:29.902525+0000","last_clean":"2024-08-17T17:37:29.902525+0000","last_became_active":"2024-08-17T17:18:55.493648+0000","last_became_peered":"2024-08-17T17:18:55.493648+0000","last_unstale":"2024-08-17T17:37:29.902525+0000","last_undegraded":"2024-08-17T17:37:29.902525+0000","last_fullsized":"2024-08-17T17:37:29.902525+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:33.301089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":447,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900531+0000","last_change":"2024-08-17T17:22:21.218491+0000","last_active":"2024-08-17T17:37:34.900531+0000","last_peered":"2024-08-17T17:37:34.900531+0000","last_clean":"2024-08-17T17:37:34.900531+0000","last_became_active":"2024-08-17T17:22:21.218143+0000","last_became_peered":"2024-08-17T17:22:21.218143+0000","last_unstale":"2024-08-17T17:37:34.900531+0000","last_undegraded":"2024-08-17T17:37:34.900531+0000","last_fullsized":"2024-08-17T17:37:34.900531+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:12.306954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.17","version":"294'10536","reported_seq":10633,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.903844+0000","last_change":"2024-08-17T17:37:28.325459+0000","last_active":"2024-08-17T17:37:34.903844+0000","last_peered":"2024-08-17T17:37:34.903844+0000","last_clean":"2024-08-17T17:37:34.903844+0000","last_became_active":"2024-08-17T17:37:24.929123+0000","last_became_peered":"2024-08-17T17:37:24.929123+0000","last_unstale":"2024-08-17T17:37:34.903844+0000","last_undegraded":"2024-08-17T17:37:34.903844+0000","last_fullsized":"2024-08-17T17:37:34.903844+0000","mapping_epoch":289,"log_start":"279'2600","ondisk_log_start":"279'2600","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"290'10429","last_scrub_stamp":"2024-08-17T17:37:28.325151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:28.325151+0000","objects_scrubbed":2429,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:27.426059+0000","scrub_duration":796,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151781376,"num_objects":2316,"num_object_clones":0,"num_object_copies":6948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":113,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856923+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:37:24.856923+0000","last_peered":"2024-08-17T17:37:24.856923+0000","last_clean":"2024-08-17T17:37:24.856923+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:37:24.856923+0000","last_undegraded":"2024-08-17T17:37:24.856923+0000","last_fullsized":"2024-08-17T17:37:24.856923+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":81,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896520+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:37:33.896520+0000","last_peered":"2024-08-17T17:37:33.896520+0000","last_clean":"2024-08-17T17:37:33.896520+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:37:33.896520+0000","last_undegraded":"2024-08-17T17:37:33.896520+0000","last_fullsized":"2024-08-17T17:37:33.896520+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.16","version":"294'10347","reported_seq":10444,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.900870+0000","last_change":"2024-08-17T17:37:27.354467+0000","last_active":"2024-08-17T17:37:34.900870+0000","last_peered":"2024-08-17T17:37:34.900870+0000","last_clean":"2024-08-17T17:37:34.900870+0000","last_became_active":"2024-08-17T17:37:24.907792+0000","last_became_peered":"2024-08-17T17:37:24.907792+0000","last_unstale":"2024-08-17T17:37:34.900870+0000","last_undegraded":"2024-08-17T17:37:34.900870+0000","last_fullsized":"2024-08-17T17:37:34.900870+0000","mapping_epoch":289,"log_start":"278'800","ondisk_log_start":"278'800","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"290'10242","last_scrub_stamp":"2024-08-17T17:37:27.354341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:27.354341+0000","objects_scrubbed":2456,"log_size":9547,"log_dups_size":0,"ondisk_log_size":9547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:33.996536+0000","scrub_duration":764,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153485312,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":114,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.863416+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:37:24.863416+0000","last_peered":"2024-08-17T17:37:24.863416+0000","last_clean":"2024-08-17T17:37:24.863416+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:37:24.863416+0000","last_undegraded":"2024-08-17T17:37:24.863416+0000","last_fullsized":"2024-08-17T17:37:24.863416+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":91,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.896536+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:37:33.896536+0000","last_peered":"2024-08-17T17:37:33.896536+0000","last_clean":"2024-08-17T17:37:33.896536+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:37:33.896536+0000","last_undegraded":"2024-08-17T17:37:33.896536+0000","last_fullsized":"2024-08-17T17:37:33.896536+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.15","version":"294'10755","reported_seq":10794,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.420447+0000","last_change":"2024-08-17T17:37:24.915609+0000","last_active":"2024-08-17T17:37:34.420447+0000","last_peered":"2024-08-17T17:37:34.420447+0000","last_clean":"2024-08-17T17:37:34.420447+0000","last_became_active":"2024-08-17T17:37:24.894215+0000","last_became_peered":"2024-08-17T17:37:24.894215+0000","last_unstale":"2024-08-17T17:37:34.420447+0000","last_undegraded":"2024-08-17T17:37:34.420447+0000","last_fullsized":"2024-08-17T17:37:34.420447+0000","mapping_epoch":289,"log_start":"278'1100","ondisk_log_start":"278'1100","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":9655,"log_dups_size":0,"ondisk_log_size":9655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151683072,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5484,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":464,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901092+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:37:33.901092+0000","last_peered":"2024-08-17T17:37:33.901092+0000","last_clean":"2024-08-17T17:37:33.901092+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:37:33.901092+0000","last_undegraded":"2024-08-17T17:37:33.901092+0000","last_fullsized":"2024-08-17T17:37:33.901092+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":465,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:33.901496+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:37:33.901496+0000","last_peered":"2024-08-17T17:37:33.901496+0000","last_clean":"2024-08-17T17:37:33.901496+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:37:33.901496+0000","last_undegraded":"2024-08-17T17:37:33.901496+0000","last_fullsized":"2024-08-17T17:37:33.901496+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"294'10515","reported_seq":10553,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.625558+0000","last_change":"2024-08-17T17:37:24.923428+0000","last_active":"2024-08-17T17:37:34.625558+0000","last_peered":"2024-08-17T17:37:34.625558+0000","last_clean":"2024-08-17T17:37:34.625558+0000","last_became_active":"2024-08-17T17:37:24.909379+0000","last_became_peered":"2024-08-17T17:37:24.909379+0000","last_unstale":"2024-08-17T17:37:34.625558+0000","last_undegraded":"2024-08-17T17:37:34.625558+0000","last_fullsized":"2024-08-17T17:37:34.625558+0000","mapping_epoch":289,"log_start":"278'300","ondisk_log_start":"278'300","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10215,"log_dups_size":0,"ondisk_log_size":10215,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151224320,"num_objects":2308,"num_object_clones":0,"num_object_copies":6924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5351,"num_write_kb":245056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":195,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.856340+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:37:24.856340+0000","last_peered":"2024-08-17T17:37:24.856340+0000","last_clean":"2024-08-17T17:37:24.856340+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:37:24.856340+0000","last_undegraded":"2024-08-17T17:37:24.856340+0000","last_fullsized":"2024-08-17T17:37:24.856340+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":486,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.901189+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:37:34.901189+0000","last_peered":"2024-08-17T17:37:34.901189+0000","last_clean":"2024-08-17T17:37:34.901189+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:37:34.901189+0000","last_undegraded":"2024-08-17T17:37:34.901189+0000","last_fullsized":"2024-08-17T17:37:34.901189+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":4856807448,"num_objects":74111,"num_object_clones":0,"num_object_copies":222333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3626,"num_objects_unfound":0,"num_objects_dirty":74111,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134529,"num_write_kb":6134881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":433,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14667493376,"data_stored":14667481160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":230883,"ondisk_log_size":230883,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201688,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2045064,"kb_used_data":1526652,"kb_used_omap":1,"kb_used_meta":518398,"kb_avail":91700088,"statfs":{"total":95995035648,"available":93900890112,"internally_reserved":0,"allocated":1563291648,"data_stored":1561736113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":530840011},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922185,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1706848,"kb_used_data":1488780,"kb_used_omap":7,"kb_used_meta":218040,"kb_avail":92038304,"statfs":{"total":95995035648,"available":94247223296,"internally_reserved":0,"allocated":1524510720,"data_stored":1522963405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":223273654},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085709,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2584988,"kb_used_data":2367500,"kb_used_omap":7,"kb_used_meta":217464,"kb_avail":91160164,"statfs":{"total":95995035648,"available":93348007936,"internally_reserved":0,"allocated":2424320000,"data_stored":2422768898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":222683577},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986598,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2127484,"kb_used_data":1528688,"kb_used_omap":1,"kb_used_meta":598782,"kb_avail":91617668,"statfs":{"total":95995035648,"available":93816492032,"internally_reserved":0,"allocated":1565376512,"data_stored":1563832635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":613153222},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435490,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2447360,"kb_used_data":1788148,"kb_used_omap":5,"kb_used_meta":659194,"kb_avail":91297792,"statfs":{"total":95995035648,"available":93488939008,"internally_reserved":0,"allocated":1831063552,"data_stored":1829527771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":675014729},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,3,3,9,9,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599016,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2518444,"kb_used_data":1828704,"kb_used_omap":7,"kb_used_meta":689720,"kb_avail":91226708,"statfs":{"total":95995035648,"available":93416148992,"internally_reserved":0,"allocated":1872592896,"data_stored":1871059976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":706273453},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762543,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2635860,"kb_used_data":2072452,"kb_used_omap":7,"kb_used_meta":563384,"kb_avail":91109292,"statfs":{"total":95995035648,"available":93295915008,"internally_reserved":0,"allocated":2122190848,"data_stored":2120654228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":576905633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,12,7,14,14,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926069,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2460632,"kb_used_data":1793100,"kb_used_omap":7,"kb_used_meta":667512,"kb_avail":91284520,"statfs":{"total":95995035648,"available":93475348480,"internally_reserved":0,"allocated":1836134400,"data_stored":1834584044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":683532892},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1824849920,"data_stored":1824849920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2118451200,"data_stored":2118451200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1858080768,"data_stored":1858076696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1827340288,"data_stored":1827340288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1547567104,"data_stored":1547567104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2419724288,"data_stored":2419720216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1518862336,"data_stored":1518862336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1552617472,"data_stored":1552613400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:37:38.542 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:37:38.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_3 pg_num 36 2024-08-17T17:37:38.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:38.995 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:39.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:39.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:38 smithi064 ceph-mon[104516]: 8.1 deep-scrub ok 2024-08-17T17:37:39.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:38 smithi064 ceph-mon[109246]: 8.1 deep-scrub ok 2024-08-17T17:37:39.126 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:39.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:39.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:39.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:39.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:39.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:39.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:38 smithi175 ceph-mon[101434]: 8.1 deep-scrub ok 2024-08-17T17:37:39.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:39.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:39.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:39.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:39.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:39.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:39.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:39.939 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:39.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:39.952 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:37:40.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:40.073 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:40.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:40.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:40.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:40.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:39 smithi064 ceph-mon[104516]: pgmap v996: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 3626/220497 objects misplaced (1.644%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:37:40.345 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:39 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:40.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:39 smithi064 ceph-mon[109246]: pgmap v996: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 3626/220497 objects misplaced (1.644%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:37:40.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:39 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:40.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:39 smithi175 ceph-mon[101434]: pgmap v996: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+clean+scrubbing+deep, 96 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 3626/220497 objects misplaced (1.644%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:37:40.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:39 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:40.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:40.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:40.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:40.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:40.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:40.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:40.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:40.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:40.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:41.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:41.038 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:41.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.181 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[104516]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:41.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:41.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[109246]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:37:41.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:41.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:41.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:40 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3814068037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:40 smithi175 ceph-mon[101434]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:37:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:37:41.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:41.514 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:41.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.660 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:41.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:41.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:41.928 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:41.931 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:41.937 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:42.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:42.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:42.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:41 smithi064 ceph-mon[104516]: pgmap v998: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 8335/210588 objects misplaced (3.958%); 12 MiB/s, 196 objects/s recovering 2024-08-17T17:37:42.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:37:42.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:41 smithi064 ceph-mon[109246]: pgmap v998: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 8335/210588 objects misplaced (3.958%); 12 MiB/s, 196 objects/s recovering 2024-08-17T17:37:42.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:37:42.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:42.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:42.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:42.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:41 smithi175 ceph-mon[101434]: pgmap v998: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 8335/210588 objects misplaced (3.958%); 12 MiB/s, 196 objects/s recovering 2024-08-17T17:37:42.429 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:37:42.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:42.466 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:42.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:42.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:42.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:42.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:42.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:42.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:42.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:42.945 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:42.955 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:42.955 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:42.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:43.061 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:37:43.061 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:42 smithi064 ceph-mon[104516]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:37:43.061 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:37:43.061 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:42 smithi064 ceph-mon[109246]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:37:43.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:43.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:43.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:43.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:43.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:37:43.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:43.345 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:43.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:37:43.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:42 smithi175 ceph-mon[101434]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:37:43.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:43.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:43.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:43.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:43.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:43.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:43.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:43.844 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:43.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:43.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:44.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:44.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:44.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[104516]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:37:44.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[104516]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8335/209649 objects misplaced (3.976%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[104516]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:43] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[109246]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[109246]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8335/209649 objects misplaced (3.976%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:37:44.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:37:44.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:43 smithi064 ceph-mon[109246]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:37:44.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:44.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:37:44.270 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:44.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:44.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:44.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:44.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:43 smithi175 ceph-mon[101434]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:37:44.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:43 smithi175 ceph-mon[101434]: pgmap v1001: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8335/209649 objects misplaced (3.976%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:37:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:37:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:37:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:43 smithi175 ceph-mon[101434]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:37:44.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:44.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:44.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:44.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:44.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:44.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:44.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:44.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:45.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:45.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:45.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:45.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:45.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.295 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[104516]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[104516]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[104516]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[109246]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[109246]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:44 smithi064 ceph-mon[109246]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:45.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:44 smithi175 ceph-mon[101434]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:44 smithi175 ceph-mon[101434]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:44 smithi175 ceph-mon[101434]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 297'10731 2024-08-17T17:37:45.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:45.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:45.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:45.672 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:45.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:45.755 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:45.755 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:45.774 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:45.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:45.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:45.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:46.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:46.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:46.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:46.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:46.173 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[104516]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:37:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[104516]: pgmap v1004: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13048/198681 objects misplaced (6.567%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:37:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[104516]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:37:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[109246]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:37:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[109246]: pgmap v1004: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13048/198681 objects misplaced (6.567%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:37:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:45 smithi064 ceph-mon[109246]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:37:46.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:46.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:46.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:46.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:45 smithi175 ceph-mon[101434]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:37:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:45 smithi175 ceph-mon[101434]: pgmap v1004: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13048/198681 objects misplaced (6.567%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:37:46.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:45 smithi175 ceph-mon[101434]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:37:46.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:46.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:46.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:46.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:46.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:46.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:46.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:46.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:46.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:47.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.080 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:47.220 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:47.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:46 smithi175 ceph-mon[101434]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:46 smithi175 ceph-mon[101434]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:46 smithi064 ceph-mon[104516]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:46 smithi064 ceph-mon[104516]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:47 smithi064 ceph-mon[109246]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:47 smithi064 ceph-mon[109246]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 298'10489 2024-08-17T17:37:47.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:47.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:47.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:47.642 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:47.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:47.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:47.913 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.014 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:48.046 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:48.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:48.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:48 smithi064 ceph-mon[104516]: pgmap v1006: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/194913 objects misplaced (5.744%); 21 MiB/s, 340 objects/s recovering 2024-08-17T17:37:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:48 smithi064 ceph-mon[109246]: pgmap v1006: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/194913 objects misplaced (5.744%); 21 MiB/s, 340 objects/s recovering 2024-08-17T17:37:48.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:48.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:48 smithi175 ceph-mon[101434]: pgmap v1006: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/194913 objects misplaced (5.744%); 21 MiB/s, 340 objects/s recovering 2024-08-17T17:37:48.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:48.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:48.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:48.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:48.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:48.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:48.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:48.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:49.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:49.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.155 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:49.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:49.256 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:49.285 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:49.387 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:49.417 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:49.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:49.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:49.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:49.694 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:49.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:49.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:49.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:49.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.064 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:50.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:50.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:50.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:50.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:50.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:50.394 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:50.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.539 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:50.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.640 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:50 smithi175 ceph-mon[101434]: pgmap v1007: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/193617 objects misplaced (5.783%); 15 MiB/s, 239 objects/s recovering 2024-08-17T17:37:50.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:50.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:50 smithi064 ceph-mon[104516]: pgmap v1007: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/193617 objects misplaced (5.783%); 15 MiB/s, 239 objects/s recovering 2024-08-17T17:37:50.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:50 smithi064 ceph-mon[109246]: pgmap v1007: 99 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11196/193617 objects misplaced (5.783%); 15 MiB/s, 239 objects/s recovering 2024-08-17T17:37:50.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:50.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:50.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:50.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:50.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:51.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.082 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:51.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:51.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:51.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:51.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:51.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.492 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.615 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:51.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:51.717 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:51.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:51.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:51.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:51.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.012 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:52.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.154 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:52.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:52.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:52.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:52.422 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:52.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:52.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:52.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:52.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:52 smithi175 ceph-mon[101434]: pgmap v1008: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11307/191313 objects misplaced (5.910%); 23 MiB/s, 367 objects/s recovering 2024-08-17T17:37:52.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:52 smithi064 ceph-mon[104516]: pgmap v1008: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11307/191313 objects misplaced (5.910%); 23 MiB/s, 367 objects/s recovering 2024-08-17T17:37:52.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:52 smithi064 ceph-mon[109246]: pgmap v1008: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11307/191313 objects misplaced (5.910%); 23 MiB/s, 367 objects/s recovering 2024-08-17T17:37:52.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:52.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:52.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:52.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.961 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:52.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:53.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:53.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:53.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:53.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:53.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:53.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:53.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:53.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:53.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:53.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:53.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:53.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:53.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:53.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:53.893 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:53.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:53.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:54.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:54.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:54.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:54.242 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:37:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:37:53] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-08-17T17:37:54.256 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:54.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:54.293 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 8 pg_num to 36 2024-08-17T17:37:54.379 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:54.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:54.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:54.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:54.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:54 smithi175 ceph-mon[101434]: pgmap v1009: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 29 MiB/s wr, 1.25k op/s; 11307/190464 objects misplaced (5.937%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:37:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/449382372' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[104516]: pgmap v1009: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 29 MiB/s wr, 1.25k op/s; 11307/190464 objects misplaced (5.937%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:37:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/449382372' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[109246]: pgmap v1009: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 29 MiB/s wr, 1.25k op/s; 11307/190464 objects misplaced (5.937%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:37:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/449382372' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:37:54.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:54.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.832 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:54.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:54.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:54.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:55.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:55.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:55.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:55.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:55.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.571 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:55.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:37:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:55 smithi175 ceph-mon[101434]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:37:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:37:55.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:37:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[104516]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:37:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:55.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:37:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:37:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[109246]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:37:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:37:55.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:55.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:55.817 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:55.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:55.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:55.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:56.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:56.077 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:56.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:56.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:56.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:56.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:56.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:56.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:56.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:56 smithi175 ceph-mon[101434]: pgmap v1011: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 1.23k op/s; 9531/183972 objects misplaced (5.181%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:37:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:37:56.562 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:56 smithi175 ceph-mon[101434]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:37:56.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:56.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[104516]: pgmap v1011: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 1.23k op/s; 9531/183972 objects misplaced (5.181%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:37:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:37:56.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[104516]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:37:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[109246]: pgmap v1011: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 1.23k op/s; 9531/183972 objects misplaced (5.181%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:37:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:37:56.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:56 smithi064 ceph-mon[109246]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:37:56.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:56.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:56.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:56.864 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:56.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:56.966 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:56.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.988 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:56.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:57.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:57.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:57.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:57.373 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:57.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:57.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:57.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:57.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:57.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:57.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:57 smithi064 ceph-mon[104516]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:37:57.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:57 smithi064 ceph-mon[104516]: 8.e scrub starts 2024-08-17T17:37:57.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:57 smithi064 ceph-mon[109246]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:37:57.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:57 smithi064 ceph-mon[109246]: 8.e scrub starts 2024-08-17T17:37:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:57 smithi175 ceph-mon[101434]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:37:57.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:57 smithi175 ceph-mon[101434]: 8.e scrub starts 2024-08-17T17:37:57.748 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:57.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:57.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:57.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:58.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:58.124 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:58.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:58.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:58.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:58.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:58.508 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:58.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:58.646 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:58 smithi175 ceph-mon[101434]: 8.3 scrub starts 2024-08-17T17:37:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:58 smithi175 ceph-mon[101434]: 8.0 scrub starts 2024-08-17T17:37:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:58 smithi175 ceph-mon[101434]: pgmap v1014: 109 pgs: 4 peering, 9 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8637/177582 objects misplaced (4.864%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:37:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:58 smithi175 ceph-mon[101434]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:37:58.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[104516]: 8.3 scrub starts 2024-08-17T17:37:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[104516]: 8.0 scrub starts 2024-08-17T17:37:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[104516]: pgmap v1014: 109 pgs: 4 peering, 9 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8637/177582 objects misplaced (4.864%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:37:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[104516]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:37:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[109246]: 8.3 scrub starts 2024-08-17T17:37:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[109246]: 8.0 scrub starts 2024-08-17T17:37:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[109246]: pgmap v1014: 109 pgs: 4 peering, 9 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8637/177582 objects misplaced (4.864%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:37:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:58 smithi064 ceph-mon[109246]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:37:58.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:58.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:58.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:58.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:58.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.062 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:59.093 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:59.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:59.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:59.378 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:59.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:59.518 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:59.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[104516]: 8.a scrub starts 2024-08-17T17:37:59.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[104516]: 8.f scrub starts 2024-08-17T17:37:59.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[104516]: 8.1e scrub starts 2024-08-17T17:37:59.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[109246]: 8.a scrub starts 2024-08-17T17:37:59.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[109246]: 8.f scrub starts 2024-08-17T17:37:59.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:37:59 smithi064 ceph-mon[109246]: 8.1e scrub starts 2024-08-17T17:37:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.653 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:59.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:59 smithi175 ceph-mon[101434]: 8.a scrub starts 2024-08-17T17:37:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:59 smithi175 ceph-mon[101434]: 8.f scrub starts 2024-08-17T17:37:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:37:59 smithi175 ceph-mon[101434]: 8.1e scrub starts 2024-08-17T17:37:59.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:59.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:37:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:59.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:59.924 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:37:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:00.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:00.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:00.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:00.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:00.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:00.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:00.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[104516]: 8.1 scrub starts 2024-08-17T17:38:00.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[104516]: 8.b scrub starts 2024-08-17T17:38:00.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[104516]: pgmap v1016: 109 pgs: 1 active+clean+scrubbing, 4 peering, 6 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 8637/175521 objects misplaced (4.921%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[109246]: 8.1 scrub starts 2024-08-17T17:38:00.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[109246]: 8.b scrub starts 2024-08-17T17:38:00.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:00 smithi064 ceph-mon[109246]: pgmap v1016: 109 pgs: 1 active+clean+scrubbing, 4 peering, 6 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 8637/175521 objects misplaced (4.921%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.482 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:00.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:00.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:00.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:00 smithi175 ceph-mon[101434]: 8.1 scrub starts 2024-08-17T17:38:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:00 smithi175 ceph-mon[101434]: 8.b scrub starts 2024-08-17T17:38:00.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:00 smithi175 ceph-mon[101434]: pgmap v1016: 109 pgs: 1 active+clean+scrubbing, 4 peering, 6 unknown, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 8637/175521 objects misplaced (4.921%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:00.757 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:00.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:01.005 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:01.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:01.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.309 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.442 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:01.600 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:01.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:01 smithi064 ceph-mon[104516]: 8.20 scrub starts 2024-08-17T17:38:01.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:38:01.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:01 smithi064 ceph-mon[109246]: 8.20 scrub starts 2024-08-17T17:38:01.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:38:01.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:01 smithi175 ceph-mon[101434]: 8.20 scrub starts 2024-08-17T17:38:01.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:38:01.704 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:01.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:38:01.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:38:01.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:38:01.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:01.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:01.732 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:01.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:01.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:38:01.758 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:01.784 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:38:01.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:01.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:01.857 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:01.985 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:01.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:02.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:02.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:02.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.325 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:38:02.343 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:02.365 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:02.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:02.614 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: 8.1a scrub starts 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: 8.1f scrub starts 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: pgmap v1017: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5685/163404 objects misplaced (3.479%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2387273884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:02.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:02 smithi175 ceph-mon[101434]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:38:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: 8.1a scrub starts 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: 8.1f scrub starts 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: pgmap v1017: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5685/163404 objects misplaced (3.479%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2387273884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[104516]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: 8.1a scrub starts 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: 8.1f scrub starts 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: pgmap v1017: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5685/163404 objects misplaced (3.479%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2387273884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1231840056' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:02 smithi064 ceph-mon[109246]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:38:02.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:02.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:02.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:02.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:03.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:03.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:03.394 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:03.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:03.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:03.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:03 smithi175 ceph-mon[101434]: 8.1a scrub starts 2024-08-17T17:38:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:03 smithi175 ceph-mon[101434]: 8.0 scrub starts 2024-08-17T17:38:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:03 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:03 smithi175 ceph-mon[101434]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:38:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:03.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[104516]: 8.1a scrub starts 2024-08-17T17:38:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[104516]: 8.0 scrub starts 2024-08-17T17:38:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:03.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[104516]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:38:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[109246]: 8.1a scrub starts 2024-08-17T17:38:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[109246]: 8.0 scrub starts 2024-08-17T17:38:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:03.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-mon[109246]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:38:03.764 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:03.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:03.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:38:03.873 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:38:03.874 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:38:03.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-08-17T17:38:03.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:03.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:04.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.093 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:04.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:03] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-08-17T17:38:04.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:04.220 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:04.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:04.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:04.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:04.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:04.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:04.634 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: 8.a scrub starts 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: 8.a scrub ok 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: 8.1b scrub starts 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: pgmap v1020: 109 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 105 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 5685/162186 objects misplaced (3.505%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: 8.1b scrub ok 2024-08-17T17:38:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:04 smithi175 ceph-mon[101434]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 306'11228 2024-08-17T17:38:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: 8.a scrub starts 2024-08-17T17:38:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: 8.a scrub ok 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: 8.1b scrub starts 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: pgmap v1020: 109 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 105 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 5685/162186 objects misplaced (3.505%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: 8.1b scrub ok 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[104516]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 306'11228 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: 8.a scrub starts 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: 8.a scrub ok 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: 8.1b scrub starts 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: pgmap v1020: 109 pgs: 1 unknown, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 105 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 5685/162186 objects misplaced (3.505%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: 8.1b scrub ok 2024-08-17T17:38:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:04 smithi064 ceph-mon[109246]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 306'11228 2024-08-17T17:38:04.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:04.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:04.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:05.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:05.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:05.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:05.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:05.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:05.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:05.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:05.401 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:38:05.402 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:38:05.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:38:05.402 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:38:05.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.402 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:134208 7.e 7:70fc098c:::benchmark_data_smithi064_210574_object35399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-17T17:38:05.403 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:38:05.403 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 331.57241535999998, 2024-08-17T17:38:05.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33046408199999999, 2024-08-17T17:38:05.403 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.403 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134208 2024-08-17T17:38:05.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336767+0000", 2024-08-17T17:38:05.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336768+0000", 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3099999999999997e-07 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336772+0000", 2024-08-17T17:38:05.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0239999999999999e-06 2024-08-17T17:38:05.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336780+0000", 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9060000000000008e-06 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336787+0000", 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9979999999999997e-06 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.336906+0000", 2024-08-17T17:38:05.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118996 2024-08-17T17:38:05.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.337017+0000", 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011017 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.337122+0000", 2024-08-17T17:38:05.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104943 2024-08-17T17:38:05.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.341658+0000", 2024-08-17T17:38:05.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045360569999999996 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.342165+0000", 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050721900000000005 2024-08-17T17:38:05.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667129+0000", 2024-08-17T17:38:05.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.324964268 2024-08-17T17:38:05.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667190+0000", 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0482000000000001e-05 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:32:31.667231+0000", 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1288000000000002e-05 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.417 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.418 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:171371 7.e 7:7e7bcdb4:::benchmark_data_smithi064_210574_object72562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-17T17:38:05.418 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:38:05.418 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 275.06134795200001, 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18653573200000001, 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.420 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171371 2024-08-17T17:38:05.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.420 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847834+0000", 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847836+0000", 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847842+0000", 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9869999999999996e-06 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847855+0000", 2024-08-17T17:38:05.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3108999999999999e-05 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.847862+0000", 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.362e-06 2024-08-17T17:38:05.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848527+0000", 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066430000000000005 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848665+0000", 2024-08-17T17:38:05.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013854899999999999 2024-08-17T17:38:05.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.848792+0000", 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012693200000000001 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860535+0000", 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011742452 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860549+0000", 2024-08-17T17:38:05.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4796e-05 2024-08-17T17:38:05.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034262+0000", 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17371302199999999 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034325+0000", 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2119000000000006e-05 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.034370+0000", 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5374999999999998e-05 2024-08-17T17:38:05.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:171373 7.e 7:79305286:::benchmark_data_smithi064_210574_object72564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-17T17:38:05.433 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 275.05655811100002, 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.191019191, 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.434 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.435 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171373 2024-08-17T17:38:05.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.435 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852624+0000", 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852626+0000", 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8309999999999999e-06 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852632+0000", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6790000000000002e-06 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852645+0000", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3906e-05 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.852658+0000", 2024-08-17T17:38:05.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2113000000000001e-05 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854505+0000", 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00184786 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854693+0000", 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018749899999999999 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.854819+0000", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012632299999999999 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860872+0000", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060525919999999999 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:27.860954+0000", 2024-08-17T17:38:05.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1791000000000003e-05 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043544+0000", 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18258986299999999 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043601+0000", 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7256999999999997e-05 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:33:28.043643+0000", 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2477e-05 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193378 7.e 7:736e894c:::benchmark_data_smithi064_210574_object94569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 239.85565950200001, 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18333747, 2024-08-17T17:38:05.442 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193378 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053523+0000", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8599999999999998e-07 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053525+0000", 2024-08-17T17:38:05.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2649999999999999e-06 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053532+0000", 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4060000000000002e-06 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.053537+0000", 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8389999999999997e-06 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.062906+0000", 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093698259999999995 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.063009+0000", 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102575 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.063126+0000", 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011680699999999999 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073581+0000", 2024-08-17T17:38:05.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010454857 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083008+0000", 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094271070000000005 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236755+0000", 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15374724200000001 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236812+0000", 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7408999999999999e-05 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236860+0000", 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7651000000000002e-05 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193384 7.e 7:79d10fe6:::benchmark_data_smithi064_210574_object94575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:38:05.451 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 239.855170246, 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18298474300000001, 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193384 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054012+0000", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054013+0000", 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054017+0000", 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.216e-06 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054028+0000", 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0789e-05 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.054034+0000", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3589999999999996e-06 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068303+0000", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014269380999999999 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068428+0000", 2024-08-17T17:38:05.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124456 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068547+0000", 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119392 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083020+0000", 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014472766999999999 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083050+0000", 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0168e-05 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236913+0000", 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.153863004 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236957+0000", 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3389000000000001e-05 2024-08-17T17:38:05.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236997+0000", 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0036000000000001e-05 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:193392 7.e 7:7ca26a87:::benchmark_data_smithi064_210574_object94583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:38:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 239.840703383, 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16874679300000001, 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193392 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:38:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068479+0000", 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068480+0000", 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.251e-06 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068487+0000", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068499+0000", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2802e-05 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.068507+0000", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1210000000000007e-06 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073606+0000", 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050989970000000001 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073707+0000", 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010047000000000001 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.073808+0000", 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010090900000000001 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083091+0000", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092827120000000003 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.083102+0000", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.095e-05 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237137+0000", 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15403577900000001 2024-08-17T17:38:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237186+0000", 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9146e-05 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237226+0000", 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9125999999999998e-05 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35217.0:193405 7.a e269/242 7:5373d8f6:::benchmark_data_smithi064_210574_object94596:head v 269'12007, mlcod=269'12005)", 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 239.82056202800001, 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16314719699999999, 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088620+0000", 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088622+0000", 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7e-06 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.088630+0000", 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5669999999999995e-06 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236740+0000", 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.148109451 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.236753+0000", 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.292e-05 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237161+0000", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040819599999999999 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.237183+0000", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2277999999999999e-05 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.251722+0000", 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014538656 2024-08-17T17:38:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:34:03.251767+0000", 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5429000000000002e-05 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:164717 8.d e288/274 8:b72ba91a:::benchmark_data_smithi064_242541_object42224:head v 288'10224, mlcod=288'10220)", 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:23.641355+0000", 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.267827203000003, 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059898527, 2024-08-17T17:38:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641355+0000", 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641355+0000", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641356+0000", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0550000000000001e-06 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641363+0000", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.922e-06 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641379+0000", 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5837000000000001e-05 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.641386+0000", 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3159999999999999e-06 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.646315+0000", 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049290580000000001 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.646339+0000", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4239999999999998e-05 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.701175+0000", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.054835436000000001 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:23.701253+0000", 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8663000000000002e-05 2024-08-17T17:38:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:166424 8.e e290/274 8:748e17b5:::benchmark_data_smithi064_242541_object43931:head v 290'10339, mlcod=290'10337)", 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:26.565010+0000", 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 36.344171961000001, 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060355434999999999, 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565010+0000", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565010+0000", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565011+0000", 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1179999999999999e-06 2024-08-17T17:38:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565019+0000", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2540000000000003e-06 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565034+0000", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5926999999999998e-05 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565042+0000", 2024-08-17T17:38:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3610000000000003e-06 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565169+0000", 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000126768 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565182+0000", 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3083e-05 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.625173+0000", 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059991764000000003 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.625366+0000", 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019216 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:166426 8.e e290/274 8:7e44fe1f:::benchmark_data_smithi064_242541_object43933:head v 290'10340, mlcod=290'10337)", 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:26.565062+0000", 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 36.344120492000002, 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.065985774999999997, 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565062+0000", 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565062+0000", 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565062+0000", 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.53e-07 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565066+0000", 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8859999999999997e-06 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565072+0000", 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9750000000000004e-06 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.565076+0000", 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3150000000000002e-06 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.569390+0000", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043137849999999997 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.569409+0000", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9219e-05 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.630832+0000", 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.061422905999999999 2024-08-17T17:38:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.631047+0000", 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000215336 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:166435 8.e e290/274 8:787784e3:::benchmark_data_smithi064_242541_object43942:head v 290'10342, mlcod=290'10337)", 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:26.590127+0000", 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 36.319055192, 2024-08-17T17:38:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059410122000000003, 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590127+0000", 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590127+0000", 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590128+0000", 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1349999999999999e-06 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590136+0000", 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3140000000000004e-06 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590152+0000", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5931999999999999e-05 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.590161+0000", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7520000000000002e-06 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.625490+0000", 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.035328443000000001 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.625537+0000", 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7794999999999999e-05 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.649376+0000", 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023839085999999999 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.649537+0000", 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016066499999999999 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:166483 8.e e290/274 8:7fc773f6:::benchmark_data_smithi064_242541_object43990:head v 290'10346, mlcod=290'10343)", 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:26.689470+0000", 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 36.219712387000001, 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.065258782000000001, 2024-08-17T17:38:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689470+0000", 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689470+0000", 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689470+0000", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3e-07 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689485+0000", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4674999999999999e-05 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689496+0000", 2024-08-17T17:38:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0471e-05 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.689504+0000", 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0260000000000007e-06 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.693399+0000", 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038952660000000001 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.693449+0000", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0411000000000002e-05 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.754508+0000", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.061058954999999998 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.754729+0000", 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022024800000000001 2024-08-17T17:38:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:166495 8.e e290/274 8:7145370c:::benchmark_data_smithi064_242541_object44002:head v 290'10347, mlcod=290'10343)", 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:26.696429+0000", 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 36.212753135, 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058437936000000003, 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696429+0000", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696429+0000", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696430+0000", 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2440000000000001e-06 2024-08-17T17:38:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696439+0000", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3250000000000008e-06 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696457+0000", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.804e-05 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.696465+0000", 2024-08-17T17:38:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7309999999999999e-06 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.699274+0000", 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028089249999999999 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.705987+0000", 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067124209999999997 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.754734+0000", 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.048747266999999997 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:26.754867+0000", 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013298300000000001 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.1.0:125687 8.1 e290/289 8:814e5c12:::benchmark_data_smithi064_242541_object69892:head v 290'10844, mlcod=290'10823)", 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:28.955197+0000", 2024-08-17T17:38:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 33.953985342000003, 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059050704000000002, 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955197+0000", 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955197+0000", 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955198+0000", 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5099999999999999e-07 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955203+0000", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.925e-06 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955214+0000", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0519e-05 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.955221+0000", 2024-08-17T17:38:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7220000000000002e-06 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.980967+0000", 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025746033000000002 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:28.980988+0000", 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1574e-05 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:29.012592+0000", 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031604132 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:29.014248+0000", 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016550479999999999 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.1.0:126271 8.1 e291/289 8:822f5310:::benchmark_data_smithi064_242541_object79906:head v 291'11154, mlcod=291'11134)", 2024-08-17T17:38:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.780458+0000", 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.128723727999997, 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.056802686999999998, 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780458+0000", 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780458+0000", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780459+0000", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6400000000000001e-07 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780466+0000", 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2790000000000004e-06 2024-08-17T17:38:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780475+0000", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5780000000000005e-06 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.780482+0000", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.272e-06 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.803063+0000", 2024-08-17T17:38:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022581014999999999 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.803088+0000", 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4414e-05 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.832121+0000", 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.029033481 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.837261+0000", 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051397839999999997 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.1.0:126278 8.1 e291/289 8:8233d7d1:::benchmark_data_smithi064_242541_object114318:head v 291'11159, mlcod=291'11134)", 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.792991+0000", 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.116190748000001, 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.065198688000000005, 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.792991+0000", 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.792991+0000", 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.792992+0000", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1900000000000001e-07 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.792998+0000", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2539999999999998e-06 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793224+0000", 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022588600000000001 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793232+0000", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4900000000000003e-06 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.837416+0000", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044183694000000003 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.837439+0000", 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3329e-05 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.857998+0000", 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020559279 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.858190+0000", 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019193699999999999 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.1.0:126279 8.1 e291/289 8:82350713:::benchmark_data_smithi064_242541_object114761:head v 291'11160, mlcod=291'11134)", 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.793261+0000", 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.115920983000002, 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068585938999999999, 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793261+0000", 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793261+0000", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793262+0000", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6899999999999997e-07 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793267+0000", 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8450000000000002e-06 2024-08-17T17:38:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793276+0000", 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9959999999999999e-06 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.793281+0000", 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1610000000000002e-06 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.841486+0000", 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.048205067999999997 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.841540+0000", 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3980000000000002e-05 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.861622+0000", 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020082408 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.861847+0000", 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022491199999999999 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:169074 8.11 e291/289 8:8c7b3b10:::benchmark_data_smithi064_242541_object46581:head v 291'10504, mlcod=291'10502)", 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.797899+0000", 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.111283252, 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060531197000000002, 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797899+0000", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797899+0000", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797900+0000", 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9800000000000002e-07 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797906+0000", 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5729999999999996e-06 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797935+0000", 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8750000000000001e-05 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797943+0000", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4390000000000003e-06 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.840525+0000", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042582383000000001 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.840546+0000", 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0849e-05 2024-08-17T17:38:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.858364+0000", 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017818665000000001 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.858430+0000", 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5640000000000002e-05 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:169108 8.1 e291/289 8:83152fbc:::benchmark_data_smithi064_242541_object46615:head v 291'11161, mlcod=291'11143)", 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.797956+0000", 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.111226313000003, 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.064275392000000001, 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797956+0000", 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797956+0000", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797956+0000", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4999999999999998e-07 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797960+0000", 2024-08-17T17:38:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.608e-06 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797967+0000", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4309999999999997e-06 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.797971+0000", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1509999999999997e-06 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.846897+0000", 2024-08-17T17:38:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.048925399000000001 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.846919+0000", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2021000000000001e-05 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.862020+0000", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015100951 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.862231+0000", 2024-08-17T17:38:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021148099999999999 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45488.0:169112 8.11 e291/289 8:89416165:::benchmark_data_smithi064_242541_object46619:head v 291'10505, mlcod=291'10502)", 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:30.804531+0000", 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.104651058999998, 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.057485894000000003, 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804531+0000", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804531+0000", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804532+0000", 2024-08-17T17:38:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0100000000000001e-06 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804538+0000", 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2230000000000004e-06 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804553+0000", 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4467000000000001e-05 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.804560+0000", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2969999999999998e-06 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.845280+0000", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040720382999999999 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.845302+0000", 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1427999999999999e-05 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.861855+0000", 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016552997999999999 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:30.862017+0000", 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016208800000000001 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:05.529 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:38:05.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:05.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:05 smithi175 ceph-mon[101434]: 8.21 scrub starts 2024-08-17T17:38:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:05 smithi175 ceph-mon[101434]: 8.21 scrub ok 2024-08-17T17:38:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:05 smithi175 ceph-mon[101434]: 8.1f scrub starts 2024-08-17T17:38:05.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:05.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[104516]: 8.21 scrub starts 2024-08-17T17:38:05.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[104516]: 8.21 scrub ok 2024-08-17T17:38:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[104516]: 8.1f scrub starts 2024-08-17T17:38:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[109246]: 8.21 scrub starts 2024-08-17T17:38:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[109246]: 8.21 scrub ok 2024-08-17T17:38:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:05 smithi064 ceph-mon[109246]: 8.1f scrub starts 2024-08-17T17:38:05.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:05.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:05.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:05.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:05.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:05.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:06.129 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:06.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:06.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:06.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:06.367 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:06.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:06.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:06.508 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: 8.1f scrub ok 2024-08-17T17:38:06.509 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: 8.1 scrub starts 2024-08-17T17:38:06.509 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: 8.1 scrub ok 2024-08-17T17:38:06.509 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: 8.20 scrub starts 2024-08-17T17:38:06.509 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: pgmap v1022: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 unknown, 1 active+remapped+backfill_wait, 105 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 2721/156441 objects misplaced (1.739%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:38:06.509 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:06 smithi175 ceph-mon[101434]: 8.20 scrub ok 2024-08-17T17:38:06.544 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:06.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: 8.1f scrub ok 2024-08-17T17:38:06.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: 8.1 scrub starts 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: 8.1 scrub ok 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: 8.20 scrub starts 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: pgmap v1022: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 unknown, 1 active+remapped+backfill_wait, 105 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 2721/156441 objects misplaced (1.739%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[104516]: 8.20 scrub ok 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: 8.1f scrub ok 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: 8.1 scrub starts 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: 8.1 scrub ok 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: 8.20 scrub starts 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: pgmap v1022: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 unknown, 1 active+remapped+backfill_wait, 105 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 2721/156441 objects misplaced (1.739%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:38:06.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:06 smithi064 ceph-mon[109246]: 8.20 scrub ok 2024-08-17T17:38:06.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:06.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:06.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:07.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:07.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:07.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:07.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:07.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:07.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:07.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:07.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:07.553 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:07.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:38:07.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:38:07.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:38:07.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:07.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:07.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:38:07.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:08.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:08.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:08.104 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:08.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:08.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:08.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.350 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:08.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:08.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:08.575 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:08.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:08.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: 8.8 scrub starts 2024-08-17T17:38:08.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: 8.f scrub starts 2024-08-17T17:38:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: pgmap v1023: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/149679 objects misplaced (2.472%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:38:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:38:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: 8.f scrub ok 2024-08-17T17:38:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[104516]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:38:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: 8.8 scrub starts 2024-08-17T17:38:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: 8.f scrub starts 2024-08-17T17:38:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: pgmap v1023: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/149679 objects misplaced (2.472%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:38:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:38:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: 8.f scrub ok 2024-08-17T17:38:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:08 smithi064 ceph-mon[109246]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:38:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: 8.8 scrub starts 2024-08-17T17:38:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: 8.f scrub starts 2024-08-17T17:38:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: pgmap v1023: 109 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/149679 objects misplaced (2.472%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:38:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:38:08.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: 8.f scrub ok 2024-08-17T17:38:08.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:08 smithi175 ceph-mon[101434]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:38:08.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:08.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:08.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.853 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:08.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:08.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:09.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:09.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:09.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:09.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:09.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:09.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:09.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:09.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:09.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:09.602 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:09.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: 8.9 scrub starts 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: 8.0 scrub starts 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: 8.0 scrub ok 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:38:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[104516]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: 8.9 scrub starts 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: 8.0 scrub starts 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: 8.0 scrub ok 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:09 smithi064 ceph-mon[109246]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:38:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: 8.9 scrub starts 2024-08-17T17:38:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: 8.0 scrub starts 2024-08-17T17:38:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: 8.0 scrub ok 2024-08-17T17:38:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:38:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:38:09.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:09 smithi175 ceph-mon[101434]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:38:09.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:09.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:09.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:09.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:09.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:10.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:10.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:38:10.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:10.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:10.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:10.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:10.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:10.420 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:10.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:10.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.546 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: 8.1a scrub starts 2024-08-17T17:38:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: 8.1a scrub ok 2024-08-17T17:38:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: pgmap v1026: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/147606 objects misplaced (2.507%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 309'11614 2024-08-17T17:38:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:10 smithi175 ceph-mon[101434]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:38:10.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:10.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: 8.1a scrub starts 2024-08-17T17:38:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: 8.1a scrub ok 2024-08-17T17:38:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: pgmap v1026: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/147606 objects misplaced (2.507%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 309'11614 2024-08-17T17:38:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[104516]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: 8.1a scrub starts 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: 8.1a scrub ok 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: pgmap v1026: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 3700/147606 objects misplaced (2.507%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 309'11614 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:10 smithi064 ceph-mon[109246]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:38:10.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:10.799 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:10.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:10.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:11.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:11.048 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:11.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:11.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:11.275 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:11.296 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:11.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:11.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:11.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:11.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.23 scrub starts 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.23 scrub ok 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.b scrub starts 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.19 scrub starts 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.2 scrub starts 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.19 scrub ok 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.b scrub ok 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.2 scrub ok 2024-08-17T17:38:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: 8.1e scrub starts 2024-08-17T17:38:11.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:38:11.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:38:11.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:11 smithi175 ceph-mon[101434]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:38:11.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.23 scrub starts 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.23 scrub ok 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.b scrub starts 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.19 scrub starts 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.2 scrub starts 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.19 scrub ok 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.b scrub ok 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.2 scrub ok 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: 8.1e scrub starts 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:38:11.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[104516]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.23 scrub starts 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.23 scrub ok 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.b scrub starts 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.19 scrub starts 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.2 scrub starts 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.19 scrub ok 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.b scrub ok 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.2 scrub ok 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:11.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: 8.1e scrub starts 2024-08-17T17:38:11.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:38:11.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:38:11.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:11 smithi064 ceph-mon[109246]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:38:11.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:11.755 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:11.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:11.815 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:11.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:11.946 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:11.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:12.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:12.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:12.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:12.200 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:12.303 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:12.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:12.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:12.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:12.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:12.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:12.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.3 scrub starts 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.3 scrub ok 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.1e scrub ok 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: pgmap v1029: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2344/141378 objects misplaced (1.658%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:12 smithi175 ceph-mon[101434]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:38:12.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.3 scrub starts 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.3 scrub ok 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.1e scrub ok 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: pgmap v1029: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2344/141378 objects misplaced (1.658%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[104516]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.3 scrub starts 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.3 scrub ok 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.1e scrub ok 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: pgmap v1029: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2344/141378 objects misplaced (1.658%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 310'11219 2024-08-17T17:38:12.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:12 smithi064 ceph-mon[109246]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:38:12.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:12.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:12.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:12.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:12.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:13.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:13.104 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:13.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:13.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:13.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[104516]: 8.c scrub starts 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[104516]: 8.c scrub ok 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[104516]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[109246]: 8.c scrub starts 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[109246]: 8.c scrub ok 2024-08-17T17:38:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:38:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:38:13.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-mon[109246]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:38:13.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:13.515 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:13.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:13.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:13.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:13.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:13 smithi175 ceph-mon[101434]: 8.c scrub starts 2024-08-17T17:38:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:13 smithi175 ceph-mon[101434]: 8.c scrub ok 2024-08-17T17:38:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:38:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:13 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:38:13.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:13 smithi175 ceph-mon[101434]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:38:13.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "pg_backfill(progress 8.16 e 316/316 lb 8:6bcfbb4e:::benchmark_data_smithi064_242541_object111121:head)", 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:13.755871+0000", 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00021476199999999999, 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000242274, 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:38:13.757 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755871+0000", 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755871+0000", 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755873+0000", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2359999999999999e-06 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755895+0000", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2112000000000001e-05 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755915+0000", 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0017000000000002e-05 2024-08-17T17:38:13.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755924+0000", 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6320000000000003e-06 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(8.16 316/313 [PushOp(8:6bd19225:::benchmark_data_smithi064_242541_object112821:head, version: 283'6961, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6bd19225:::benchmark_data_smithi064_242541_object112821:head@283'6961, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:13.755946+0000", 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00014017899999999999, 2024-08-17T17:38:13.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023735499999999999, 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755946+0000", 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755946+0000", 2024-08-17T17:38:13.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.755946+0000", 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5199999999999998e-07 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.756012+0000", 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5512000000000006e-05 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.756028+0000", 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6342999999999999e-05 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:13.756043+0000", 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4713e-05 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:13.763 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:38:13.764 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 2 2024-08-17T17:38:13.764 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:38:13.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:13.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:13.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:14.009 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:13] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-08-17T17:38:14.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:14.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:14.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:14.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:14.175 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:14.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:14.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:14.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:14.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:14.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:14.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.13 scrub starts 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.10 scrub starts 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: pgmap v1032: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 534/140028 objects misplaced (0.381%); 47 MiB/s, 754 objects/s recovering 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.13 scrub ok 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.5 scrub starts 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.5 scrub ok 2024-08-17T17:38:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.e scrub starts 2024-08-17T17:38:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: 8.e scrub ok 2024-08-17T17:38:14.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:14 smithi175 ceph-mon[101434]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:38:14.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.13 scrub starts 2024-08-17T17:38:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.10 scrub starts 2024-08-17T17:38:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: pgmap v1032: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 534/140028 objects misplaced (0.381%); 47 MiB/s, 754 objects/s recovering 2024-08-17T17:38:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.13 scrub ok 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.5 scrub starts 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.5 scrub ok 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.e scrub starts 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: 8.e scrub ok 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[104516]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.13 scrub starts 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.10 scrub starts 2024-08-17T17:38:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: pgmap v1032: 109 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 106 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 534/140028 objects misplaced (0.381%); 47 MiB/s, 754 objects/s recovering 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.13 scrub ok 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 313'11404 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.5 scrub starts 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.5 scrub ok 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.e scrub starts 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: 8.e scrub ok 2024-08-17T17:38:14.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:14 smithi064 ceph-mon[109246]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:38:14.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:14.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:14.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:38:14.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:14.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:14.921 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:14.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:15.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:38:15.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:15.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:15.066 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:38:15.067 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:15.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:15.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:15.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:38:15.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:15.116 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:38:15.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:15.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:38:15.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:15.164 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:38:15.165 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:15.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:15.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:38:15.191 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:15.198 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:15.214 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:38:15.214 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:15.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:15.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:15.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:15.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:15.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:15.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:15.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[104516]: 8.22 scrub starts 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1841116314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[104516]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[109246]: 8.22 scrub starts 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1841116314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:38:15.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:15 smithi064 ceph-mon[109246]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:38:15.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:15.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:15.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:15.862 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:15.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:15 smithi175 ceph-mon[101434]: 8.22 scrub starts 2024-08-17T17:38:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:15 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1841116314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:38:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:15 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:38:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:15 smithi175 ceph-mon[101434]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:38:15.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:15.993 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:16.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:16.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:16.219 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:16.241 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:16.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:16.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:16.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:16.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:16.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:16.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[104516]: pgmap v1035: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1443/138522 objects misplaced (1.042%); 28 MiB/s, 444 objects/s recovering 2024-08-17T17:38:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[104516]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 315'11411 2024-08-17T17:38:16.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[104516]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:38:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[109246]: pgmap v1035: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1443/138522 objects misplaced (1.042%); 28 MiB/s, 444 objects/s recovering 2024-08-17T17:38:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[109246]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 315'11411 2024-08-17T17:38:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:16 smithi064 ceph-mon[109246]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:38:16.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:16.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:16.840 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:16.841 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:16.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:16.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:16.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:16 smithi175 ceph-mon[101434]: pgmap v1035: 109 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 106 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1443/138522 objects misplaced (1.042%); 28 MiB/s, 444 objects/s recovering 2024-08-17T17:38:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:16 smithi175 ceph-mon[101434]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 315'11411 2024-08-17T17:38:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:16 smithi175 ceph-mon[101434]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:38:16.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.012 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.031 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2024-08-17T17:38:17.031 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2024-08-17T17:38:17.031 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.68909 -> 3.68909 (expected baseline 5.98044) 2024-08-17T17:38:17.031 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 36 -> 36 pgs (0.880734 -> 0.880734 * mean) 2024-08-17T17:38:17.031 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 47 -> 47 pgs (1.14985 -> 1.14985 * mean) 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0195 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.1948 2024-08-17T17:38:17.032 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2024-08-17T17:38:17.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:17.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:17.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:17.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.493 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:17.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:17.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:17.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:17.937 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:18.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:18.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:38:18.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:18.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:18.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[104516]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[104516]: 8.4 scrub starts 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[104516]: pgmap v1038: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2351/136359 objects misplaced (1.724%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[104516]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 317'11223 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[104516]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[109246]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[109246]: 8.4 scrub starts 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[109246]: pgmap v1038: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2351/136359 objects misplaced (1.724%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[109246]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 317'11223 2024-08-17T17:38:18.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:18 smithi064 ceph-mon[109246]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:38:18.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:18.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:18.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:18.623 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:18 smithi175 ceph-mon[101434]: from='client.45680 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:18 smithi175 ceph-mon[101434]: 8.4 scrub starts 2024-08-17T17:38:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:18 smithi175 ceph-mon[101434]: pgmap v1038: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2351/136359 objects misplaced (1.724%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T17:38:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:18 smithi175 ceph-mon[101434]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 317'11223 2024-08-17T17:38:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:18 smithi175 ceph-mon[101434]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:38:18.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:18.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:18.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:18.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:18.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:19.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:19.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:19.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:19.266 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:19.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:19.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:19.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:19.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:19 smithi175 ceph-mon[101434]: 8.12 scrub starts 2024-08-17T17:38:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:19 smithi175 ceph-mon[101434]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:38:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:19 smithi175 ceph-mon[101434]: 8.12 scrub ok 2024-08-17T17:38:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:19 smithi175 ceph-mon[101434]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:38:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[104516]: 8.12 scrub starts 2024-08-17T17:38:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[104516]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:38:19.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[104516]: 8.12 scrub ok 2024-08-17T17:38:19.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[104516]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:38:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[109246]: 8.12 scrub starts 2024-08-17T17:38:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[109246]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:38:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[109246]: 8.12 scrub ok 2024-08-17T17:38:19.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:19 smithi064 ceph-mon[109246]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:38:19.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:19.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:20.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:20.097 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:20.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:20.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:20.336 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.372 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:20.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:20.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:20.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:20.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:20.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:20 smithi175 ceph-mon[101434]: pgmap v1041: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 1724/134325 objects misplaced (1.283%); 42 MiB/s, 670 objects/s recovering 2024-08-17T17:38:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:20 smithi064 ceph-mon[104516]: pgmap v1041: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 1724/134325 objects misplaced (1.283%); 42 MiB/s, 670 objects/s recovering 2024-08-17T17:38:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:20 smithi064 ceph-mon[109246]: pgmap v1041: 109 pgs: 1 peering, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 105 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 349 op/s; 1724/134325 objects misplaced (1.283%); 42 MiB/s, 670 objects/s recovering 2024-08-17T17:38:20.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:20.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:20.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.916 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:21.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:21.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:21.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:21.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:21.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:21.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:21.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:21.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:21.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:21.550 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:21.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:21.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:21 smithi175 ceph-mon[101434]: 8.1c scrub starts 2024-08-17T17:38:21.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:21 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:38:21.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:21 smithi064 ceph-mon[104516]: 8.1c scrub starts 2024-08-17T17:38:21.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:21 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:38:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:21 smithi064 ceph-mon[109246]: 8.1c scrub starts 2024-08-17T17:38:21.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:21 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:38:21.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:21.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:21.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:21.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:22.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:22.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:22.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:22.345 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:22.447 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:22.470 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:22.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:22.594 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:22.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:22 smithi175 ceph-mon[101434]: 8.1c scrub ok 2024-08-17T17:38:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:22 smithi175 ceph-mon[101434]: pgmap v1043: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 2820/128178 objects misplaced (2.200%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:38:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:38:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:22 smithi175 ceph-mon[101434]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[104516]: 8.1c scrub ok 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[104516]: pgmap v1043: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 2820/128178 objects misplaced (2.200%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[104516]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[109246]: 8.1c scrub ok 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[109246]: pgmap v1043: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 2820/128178 objects misplaced (2.200%); 22 MiB/s, 354 objects/s recovering 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:38:22.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:22 smithi064 ceph-mon[109246]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:38:22.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:22.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:23.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:23.096 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:23.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:23.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:23.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:23.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:23.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:23.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:23.469 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:23.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:23.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:23 smithi175 ceph-mon[101434]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:38:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:38:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:23 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:38:23.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:23 smithi175 ceph-mon[101434]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:38:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[104516]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:38:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:38:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:38:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[104516]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:38:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[109246]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:38:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:38:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:38:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-mon[109246]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:38:23.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:23.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:23.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:23.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:23.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:23.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:23.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:24.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:24.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:23] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-08-17T17:38:24.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.234 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:24.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:24.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:24.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:24 smithi175 ceph-mon[101434]: pgmap v1046: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1378/125526 objects misplaced (1.098%); 10 MiB/s, 160 objects/s recovering 2024-08-17T17:38:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:24 smithi175 ceph-mon[101434]: 8.1a starting backfill to osd.0 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:24 smithi175 ceph-mon[101434]: 8.1a starting backfill to osd.6 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:24 smithi175 ceph-mon[101434]: 8.1a starting backfill to osd.7 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:24 smithi175 ceph-mon[101434]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:38:24.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[104516]: pgmap v1046: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1378/125526 objects misplaced (1.098%); 10 MiB/s, 160 objects/s recovering 2024-08-17T17:38:24.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[104516]: 8.1a starting backfill to osd.0 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[104516]: 8.1a starting backfill to osd.6 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[104516]: 8.1a starting backfill to osd.7 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[104516]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:38:24.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[109246]: pgmap v1046: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1378/125526 objects misplaced (1.098%); 10 MiB/s, 160 objects/s recovering 2024-08-17T17:38:24.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[109246]: 8.1a starting backfill to osd.0 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[109246]: 8.1a starting backfill to osd.6 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[109246]: 8.1a starting backfill to osd.7 from (0'0,0'0] MAX to 322'12165 2024-08-17T17:38:24.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:24 smithi064 ceph-mon[109246]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:38:24.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:24.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:24.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.994 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:25.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:25.120 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:25.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:25.247 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:25.349 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:25.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: 8.14 scrub starts 2024-08-17T17:38:25.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: 8.14 scrub ok 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"} : dispatch 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"}]': finished 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[104516]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: 8.14 scrub starts 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: 8.14 scrub ok 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:25.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"} : dispatch 2024-08-17T17:38:25.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"}]': finished 2024-08-17T17:38:25.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:25 smithi064 ceph-mon[109246]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:38:25.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:25.518 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:25.619 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:25.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: 8.14 scrub starts 2024-08-17T17:38:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: 8.14 scrub ok 2024-08-17T17:38:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"} : dispatch 2024-08-17T17:38:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "29"}]': finished 2024-08-17T17:38:25.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:25 smithi175 ceph-mon[101434]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:38:25.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:25.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:25.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:25.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:25.964 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:38:25.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-08-17T17:38:26.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:26.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:26.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:26.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:26.169 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:26.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:26.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:26.294 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:26.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:26.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:26.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:26.550 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:26.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:26.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:26.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: pgmap v1049: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 107 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1096/121677 objects misplaced (0.901%) 2024-08-17T17:38:26.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: 8.22 scrub starts 2024-08-17T17:38:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: 8.1b starting backfill to osd.2 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: 8.1b starting backfill to osd.6 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: 8.22 scrub ok 2024-08-17T17:38:26.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:26 smithi175 ceph-mon[101434]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:38:26.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: pgmap v1049: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 107 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1096/121677 objects misplaced (0.901%) 2024-08-17T17:38:26.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: 8.22 scrub starts 2024-08-17T17:38:26.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: 8.1b starting backfill to osd.2 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: 8.1b starting backfill to osd.6 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: 8.22 scrub ok 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[104516]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: pgmap v1049: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 107 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 1096/121677 objects misplaced (0.901%) 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: 8.22 scrub starts 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: 8.1b starting backfill to osd.2 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: 8.1b starting backfill to osd.6 from (0'0,0'0] MAX to 323'12334 2024-08-17T17:38:26.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: 8.22 scrub ok 2024-08-17T17:38:26.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:26 smithi064 ceph-mon[109246]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:38:26.679 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:26.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:26.806 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:26.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:26.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:27.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:27.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:27.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:27.181 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:27.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:27.305 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:27.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:27.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:27.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:27.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:27.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:27.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:27.702 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:27.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:27.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:27.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:27.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:27.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:28.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:28.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:28.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:28.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:28.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:28.333 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:28.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:28.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:28.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:28.612 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: 8.d scrub starts 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: 8.d scrub ok 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: pgmap v1052: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/114363 objects misplaced (3.224%); 12 MiB/s, 189 objects/s recovering 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: 8.1c starting backfill to osd.4 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:38:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:28 smithi175 ceph-mon[101434]: 8.1c starting backfill to osd.5 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: 8.d scrub starts 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: 8.d scrub ok 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: pgmap v1052: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/114363 objects misplaced (3.224%); 12 MiB/s, 189 objects/s recovering 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: 8.1c starting backfill to osd.4 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[104516]: 8.1c starting backfill to osd.5 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: 8.d scrub starts 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: 8.d scrub ok 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: pgmap v1052: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/114363 objects misplaced (3.224%); 12 MiB/s, 189 objects/s recovering 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: 8.1c starting backfill to osd.4 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:38:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:28 smithi064 ceph-mon[109246]: 8.1c starting backfill to osd.5 from (0'0,0'0] MAX to 326'12196 2024-08-17T17:38:28.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:28.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:28.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:28.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:28.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:29.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:29.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:29.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:29.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.279 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:29.380 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:29.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:29.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:29.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:29 smithi064 ceph-mon[104516]: 8.15 scrub starts 2024-08-17T17:38:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:29 smithi064 ceph-mon[104516]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:38:29.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:29 smithi064 ceph-mon[109246]: 8.15 scrub starts 2024-08-17T17:38:29.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:29 smithi064 ceph-mon[109246]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:38:29.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:29.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:29 smithi175 ceph-mon[101434]: 8.15 scrub starts 2024-08-17T17:38:29.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:29 smithi175 ceph-mon[101434]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:38:29.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:29.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:29.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:29.921 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:29.960 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:30.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:30.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:30.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:30.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.249 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:30.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:30.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:30.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:30.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:30.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:30.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:30.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:30 smithi064 ceph-mon[104516]: pgmap v1055: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/113163 objects misplaced (3.258%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:38:30.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:30 smithi064 ceph-mon[109246]: pgmap v1055: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/113163 objects misplaced (3.258%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:38:30.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:30.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:30 smithi175 ceph-mon[101434]: pgmap v1055: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3687/113163 objects misplaced (3.258%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T17:38:30.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:30.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:30.670 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:30.670 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:30.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:30.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:30.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:30.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:30.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:31.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:31.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:31.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:31.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:31.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:31.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:31.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:31 smithi064 ceph-mon[104516]: 8.10 scrub starts 2024-08-17T17:38:31.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"} : dispatch 2024-08-17T17:38:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:31 smithi064 ceph-mon[109246]: 8.10 scrub starts 2024-08-17T17:38:31.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"} : dispatch 2024-08-17T17:38:31.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:31.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:31.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:31.637 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:31 smithi175 ceph-mon[101434]: 8.10 scrub starts 2024-08-17T17:38:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"} : dispatch 2024-08-17T17:38:31.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:31.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:31.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:31.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:31.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:31.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:32.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:32.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:32.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:32.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:38:32.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:32.164 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:32.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:32.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:32.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:32.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:32.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:32.542 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:32.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:32.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:32 smithi175 ceph-mon[101434]: pgmap v1056: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 3762/110979 objects misplaced (3.390%); 15 MiB/s, 238 objects/s recovering 2024-08-17T17:38:32.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:32 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"}]': finished 2024-08-17T17:38:32.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:32 smithi175 ceph-mon[101434]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:38:32.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[104516]: pgmap v1056: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 3762/110979 objects misplaced (3.390%); 15 MiB/s, 238 objects/s recovering 2024-08-17T17:38:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"}]': finished 2024-08-17T17:38:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[104516]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:38:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[109246]: pgmap v1056: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 3762/110979 objects misplaced (3.390%); 15 MiB/s, 238 objects/s recovering 2024-08-17T17:38:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "30"}]': finished 2024-08-17T17:38:32.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:32 smithi064 ceph-mon[109246]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:38:32.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:32.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:32.892 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:32.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:32.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:33.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:33.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:33.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:33.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:33.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.284 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.386 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:33.407 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:33.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:33.530 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:33.632 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:33.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:33 smithi175 ceph-mon[101434]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:38:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:33 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:38:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:33 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:38:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:33 smithi175 ceph-mon[101434]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:38:33.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.662 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:33.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[104516]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:38:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:38:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:38:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[104516]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:38:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[109246]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:38:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:38:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:38:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-mon[109246]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:38:33.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:33.806 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:33.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:33.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:33.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:34.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:34.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:34.101 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:33] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-08-17T17:38:34.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:34.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:34.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:34.307 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:34.317 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:38:34.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:34.435 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:34.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:34.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: pgmap v1059: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1504/107616 objects misplaced (1.398%); 21 MiB/s, 336 objects/s recovering 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: 8.1d starting backfill to osd.0 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: 8.1d starting backfill to osd.1 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: 8.1d starting backfill to osd.3 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3467836075' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:38:34.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:34.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:34 smithi175 ceph-mon[101434]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:38:34.668 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: pgmap v1059: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1504/107616 objects misplaced (1.398%); 21 MiB/s, 336 objects/s recovering 2024-08-17T17:38:34.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: 8.1d starting backfill to osd.0 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: 8.1d starting backfill to osd.1 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: 8.1d starting backfill to osd.3 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3467836075' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[104516]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: pgmap v1059: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 1504/107616 objects misplaced (1.398%); 21 MiB/s, 336 objects/s recovering 2024-08-17T17:38:34.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: 8.1d starting backfill to osd.0 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: 8.1d starting backfill to osd.1 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: 8.1d starting backfill to osd.3 from (0'0,0'0] MAX to 330'12311 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3467836075' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:34.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:34 smithi064 ceph-mon[109246]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:38:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:34.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:34.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:34.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:34.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:35.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:35.063 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:35.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:35.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:35.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:35.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:35.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:35.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[104516]: 8.11 scrub starts 2024-08-17T17:38:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[104516]: 8.11 scrub ok 2024-08-17T17:38:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[104516]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:38:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[109246]: 8.11 scrub starts 2024-08-17T17:38:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[109246]: 8.11 scrub ok 2024-08-17T17:38:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:35 smithi064 ceph-mon[109246]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:38:35.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:35.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:35.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:35.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:35 smithi175 ceph-mon[101434]: 8.11 scrub starts 2024-08-17T17:38:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:35 smithi175 ceph-mon[101434]: 8.11 scrub ok 2024-08-17T17:38:35.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:35 smithi175 ceph-mon[101434]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:38:35.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:35.692 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:38:35.692 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:35.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:35.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:38:35.729 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:35.740 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:35.770 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:38:35.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:35.799 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:35.804 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:38:35.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:35.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:35.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:38:35.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:35.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:35.868 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:38:35.869 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:35.903 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:38:35.903 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:35.939 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:38:35.939 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:35.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:35.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:36.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:36.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:36.262 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:36.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:36.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:36.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:36.558 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:36.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: 8.8 deep-scrub starts 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: pgmap v1062: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 1504/100440 objects misplaced (1.497%); 17 MiB/s, 269 objects/s recovering 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: 8.18 scrub starts 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: 8.1e starting backfill to osd.3 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: 8.1e starting backfill to osd.7 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/229513948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: 8.8 deep-scrub starts 2024-08-17T17:38:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: pgmap v1062: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 1504/100440 objects misplaced (1.497%); 17 MiB/s, 269 objects/s recovering 2024-08-17T17:38:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: 8.18 scrub starts 2024-08-17T17:38:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: 8.1e starting backfill to osd.3 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: 8.1e starting backfill to osd.7 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/229513948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: 8.8 deep-scrub starts 2024-08-17T17:38:36.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: pgmap v1062: 109 pgs: 1 unknown, 2 active+remapped+backfilling, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 1504/100440 objects misplaced (1.497%); 17 MiB/s, 269 objects/s recovering 2024-08-17T17:38:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: 8.18 scrub starts 2024-08-17T17:38:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: 8.1e starting backfill to osd.3 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: 8.1e starting backfill to osd.7 from (0'0,0'0] MAX to 331'12335 2024-08-17T17:38:36.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/229513948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:36.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:36.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:36.938 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.973 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:36.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:37.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:37.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:38:37.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:37.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:37.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.393 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:37.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:37.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:37.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:37 smithi175 ceph-mon[101434]: 8.18 scrub ok 2024-08-17T17:38:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:37 smithi175 ceph-mon[101434]: 8.9 deep-scrub starts 2024-08-17T17:38:37.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:37 smithi064 ceph-mon[104516]: 8.18 scrub ok 2024-08-17T17:38:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:37 smithi064 ceph-mon[104516]: 8.9 deep-scrub starts 2024-08-17T17:38:37.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:37 smithi064 ceph-mon[109246]: 8.18 scrub ok 2024-08-17T17:38:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:37 smithi064 ceph-mon[109246]: 8.9 deep-scrub starts 2024-08-17T17:38:37.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:37.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:37.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:37.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:37.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:38.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:38.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:38.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.201 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:38.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:38.302 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:38.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:38.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:38.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:38.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[109246]: 8.4 scrub starts 2024-08-17T17:38:38.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[109246]: pgmap v1064: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/96465 objects misplaced (1.057%) 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[109246]: 8.4 scrub ok 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[109246]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[104516]: 8.4 scrub starts 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[104516]: pgmap v1064: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/96465 objects misplaced (1.057%) 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[104516]: 8.4 scrub ok 2024-08-17T17:38:38.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:38 smithi064 ceph-mon[104516]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:38:38.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:38.607 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:38 smithi175 ceph-mon[101434]: 8.4 scrub starts 2024-08-17T17:38:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:38 smithi175 ceph-mon[101434]: pgmap v1064: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/96465 objects misplaced (1.057%) 2024-08-17T17:38:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:38 smithi175 ceph-mon[101434]: 8.4 scrub ok 2024-08-17T17:38:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:38 smithi175 ceph-mon[101434]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:38:38.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:38.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:38.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:38.925 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:39.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:39.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:39.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:39.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:39.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:39.307 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:39.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:39.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:39.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:39.579 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:39.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:39.717 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:39.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:39.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:39.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:40.000 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:40.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.151 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:40.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:40.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:40.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:40.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:40.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:40.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: pgmap v1066: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/94995 objects misplaced (1.074%) 2024-08-17T17:38:40.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T17:38:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: 8.1d deep-scrub starts 2024-08-17T17:38:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:40.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: pgmap v1066: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/94995 objects misplaced (1.074%) 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: 8.1d deep-scrub starts 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:40.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:40.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:40.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:40.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: pgmap v1066: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 unknown, 106 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 1020/94995 objects misplaced (1.074%) 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: 8.1d deep-scrub starts 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:40.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:40.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:40.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:40.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:40.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:41.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:41.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:41.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:41.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:41.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:41.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"} : dispatch 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:41.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"} : dispatch 2024-08-17T17:38:41.443 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:41.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:41.571 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:41.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"} : dispatch 2024-08-17T17:38:41.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:41.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:41.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:41.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:41.840 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2024-08-17T17:38:41.840 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2024-08-17T17:38:41.840 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.9191 -> 3.9191 (expected baseline 5.98044) 2024-08-17T17:38:41.840 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 33 -> 33 pgs (0.807339 -> 0.807339 * mean) 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 45 -> 45 pgs (1.10092 -> 1.10092 * mean) 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0160 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.1595 2024-08-17T17:38:41.841 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2024-08-17T17:38:41.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:41.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:41.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:42.086 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:42.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:42.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:42.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:42.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:42.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:42.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:42.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.625 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:42.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:42.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:42.881 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:42 smithi175 ceph-mon[101434]: pgmap v1068: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 776/92760 objects misplaced (0.837%) 2024-08-17T17:38:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:42 smithi175 ceph-mon[101434]: 8.8 deep-scrub ok 2024-08-17T17:38:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:42 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"}]': finished 2024-08-17T17:38:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:42 smithi175 ceph-mon[101434]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:38:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[104516]: pgmap v1068: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 776/92760 objects misplaced (0.837%) 2024-08-17T17:38:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[104516]: 8.8 deep-scrub ok 2024-08-17T17:38:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"}]': finished 2024-08-17T17:38:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[104516]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:38:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[109246]: pgmap v1068: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 776/92760 objects misplaced (0.837%) 2024-08-17T17:38:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[109246]: 8.8 deep-scrub ok 2024-08-17T17:38:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "32"}]': finished 2024-08-17T17:38:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:42 smithi064 ceph-mon[109246]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:38:42.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:43.004 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:43.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:43.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:43.257 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:43.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:43.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:43.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:43.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:43.653 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:43.756 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:43.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:43.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:43.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:43.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: 8.9 deep-scrub ok 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"} : dispatch 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"}]': finished 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: 8.1f starting backfill to osd.0 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: 8.1f starting backfill to osd.5 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.914 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:43 smithi175 ceph-mon[101434]: 8.1f starting backfill to osd.7 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:43.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: 8.9 deep-scrub ok 2024-08-17T17:38:43.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"} : dispatch 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"}]': finished 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: 8.1f starting backfill to osd.0 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: 8.1f starting backfill to osd.5 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[109246]: 8.1f starting backfill to osd.7 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: from='client.45698 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: 8.9 deep-scrub ok 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"} : dispatch 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "33"}]': finished 2024-08-17T17:38:43.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:38:43.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: 8.1f starting backfill to osd.0 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: 8.1f starting backfill to osd.5 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:43.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-mon[104516]: 8.1f starting backfill to osd.7 from (0'0,0'0] MAX to 337'12641 2024-08-17T17:38:44.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.057 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:44.057 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:44.068 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:38:44.068 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:38:44.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96080 7.c 7:3f226a37:::benchmark_data_smithi064_210574_object96079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 428.919357026, 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2216669489999998, 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:44.069 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96080 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145553+0000", 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145554+0000", 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2600000000000001e-07 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145578+0000", 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4579999999999998e-05 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145587+0000", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5979999999999997e-06 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145593+0000", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7620000000000001e-06 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145700+0000", 2024-08-17T17:38:44.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010727500000000001 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145831+0000", 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013128200000000001 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145967+0000", 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013590600000000001 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163850+0000", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017883087999999998 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163863+0000", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3019e-05 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149735+0000", 2024-08-17T17:38:44.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98587233299999999 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149749+0000", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3543999999999999e-05 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367152+0000", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.217402664 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.367220+0000", 2024-08-17T17:38:44.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7972000000000006e-05 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96087 7.c 7:347c5818:::benchmark_data_smithi064_210574_object96086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 428.91384729399999, 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.232266434, 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:38:44.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96087 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151062+0000", 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151064+0000", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6819999999999999e-06 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151095+0000", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0735999999999998e-05 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151106+0000", 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1232e-05 2024-08-17T17:38:44.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.151129+0000", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2767000000000001e-05 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163899+0000", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012769951 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.163907+0000", 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.354e-06 2024-08-17T17:38:44.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149776+0000", 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98586885099999999 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.149784+0000", 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2039999999999997e-06 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370442+0000", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.220658105 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370625+0000", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018222600000000001 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.370817+0000", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019209699999999999 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.373489+0000", 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026719090000000001 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.375180+0000", 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016911890000000001 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380081+0000", 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049010909999999998 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.380117+0000", 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6176000000000001e-05 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.383329+0000", 2024-08-17T17:38:44.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003211864 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:164891 8.10 8:0fbc6c8f:::benchmark_data_smithi064_242541_object42398:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:24.038807+0000", 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.026102913000003, 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92275183000000005, 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164891 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038807+0000", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038807+0000", 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038809+0000", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3999999999999999e-06 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038817+0000", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1349999999999992e-06 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038840+0000", 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2206999999999999e-05 2024-08-17T17:38:44.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.038850+0000", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0276e-05 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.254975+0000", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21612546199999999 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.254986+0000", 2024-08-17T17:38:44.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0455000000000001e-05 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.958315+0000", 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70332933099999995 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.958454+0000", 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138419 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.961458+0000", 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003004301 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:24.961559+0000", 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100844 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:197983 8.16 8.c7c3c9d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.453980+0000", 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.610930025999998, 2024-08-17T17:38:44.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86114401900000004, 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 197983 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.453980+0000", 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.453980+0000", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.453981+0000", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3039999999999999e-06 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.453986+0000", 2024-08-17T17:38:44.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.417e-06 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.453997+0000", 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1e-05 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.454003+0000", 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9220000000000002e-06 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.454059+0000", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6073999999999999e-05 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.454070+0000", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0835999999999999e-05 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315112+0000", 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86104191600000002 2024-08-17T17:38:44.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315124+0000", 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.155e-05 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198044 8.16 8.aecb0ff6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.506478+0000", 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.558431390999999, 2024-08-17T17:38:44.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80873251199999996, 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198044 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506478+0000", 2024-08-17T17:38:44.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506478+0000", 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506480+0000", 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.827e-06 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506486+0000", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7810000000000002e-06 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506503+0000", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7365000000000001e-05 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506511+0000", 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.435e-06 2024-08-17T17:38:44.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506580+0000", 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8777999999999998e-05 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.506587+0000", 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6729999999999992e-06 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315203+0000", 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80861536099999998 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315211+0000", 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2919999999999996e-06 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.099 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198050 8.16 8.637d9b16 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.507940+0000", 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.556969752999997, 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80730236300000002, 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198050 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507940+0000", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507940+0000", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507941+0000", 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.006e-06 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507946+0000", 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7899999999999999e-06 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507958+0000", 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1813e-05 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.507965+0000", 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4059999999999999e-06 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.508019+0000", 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3813000000000001e-05 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.508026+0000", 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315235+0000", 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80720851000000005 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315242+0000", 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6249999999999998e-06 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198079 8.16 8.7ec0f8d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.527973+0000", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.536937131999998, 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.787300373, 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198079 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527973+0000", 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527973+0000", 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527974+0000", 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.438e-06 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527978+0000", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2250000000000002e-06 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527988+0000", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0203e-05 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.527996+0000", 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0639999999999996e-06 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.528058+0000", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2538000000000003e-05 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.528066+0000", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4410000000000004e-06 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315265+0000", 2024-08-17T17:38:44.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78719987899999999 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315273+0000", 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5850000000000002e-06 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198108 8.16 8.75218876 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.552023+0000", 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.512886715999997, 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76328068000000004, 2024-08-17T17:38:44.109 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198108 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552023+0000", 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552023+0000", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552025+0000", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.624e-06 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552030+0000", 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7440000000000004e-06 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552049+0000", 2024-08-17T17:38:44.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8309e-05 2024-08-17T17:38:44.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552057+0000", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552113+0000", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6332000000000002e-05 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.552125+0000", 2024-08-17T17:38:44.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2057e-05 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315296+0000", 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76317098299999997 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315304+0000", 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2289999999999997e-06 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198137 8.16 8.3830ba96 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.579536+0000", 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.485373713000001, 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73579833800000005, 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.115 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198137 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579536+0000", 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579536+0000", 2024-08-17T17:38:44.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579538+0000", 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1710000000000001e-06 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579545+0000", 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9480000000000002e-06 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579564+0000", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8898000000000001e-05 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579573+0000", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8310000000000007e-06 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579635+0000", 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2096999999999999e-05 2024-08-17T17:38:44.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.579643+0000", 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5199999999999997e-06 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315327+0000", 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73568345300000004 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315334+0000", 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4200000000000001e-06 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198151 8.16 8.7a8625d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.589535+0000", 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.475374338999998, 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72582928599999996, 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.121 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198151 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589535+0000", 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589535+0000", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589537+0000", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3289999999999999e-06 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589547+0000", 2024-08-17T17:38:44.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0251999999999999e-05 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589565+0000", 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7679000000000001e-05 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589573+0000", 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2549999999999997e-06 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589634+0000", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1310999999999996e-05 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.589642+0000", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1790000000000008e-06 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315357+0000", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72571467199999995 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315365+0000", 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6090000000000003e-06 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198172 8.16 8.48f5f296 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:09.608538+0000", 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 34.45637163, 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70685785599999995, 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:38:44.126 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198172 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608538+0000", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608538+0000", 2024-08-17T17:38:44.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608539+0000", 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001e-06 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608543+0000", 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8800000000000001e-06 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608555+0000", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2128e-05 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608560+0000", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608614+0000", 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.363e-05 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:09.608622+0000", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3979999999999999e-06 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315388+0000", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70676590800000005 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.315396+0000", 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6809999999999998e-06 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.130 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198372 8.13 8:ce279da2:::benchmark_data_smithi064_242541_object75879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.372010+0000", 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.692899369000003, 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98032647900000003, 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198372 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.131 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372010+0000", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372010+0000", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372011+0000", 2024-08-17T17:38:44.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372016+0000", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5449999999999997e-06 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372025+0000", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4660000000000005e-06 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372031+0000", 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6999999999999996e-06 2024-08-17T17:38:44.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372083+0000", 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1499000000000003e-05 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.372093+0000", 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0047000000000001e-05 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.347241+0000", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97514875000000001 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.347369+0000", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127279 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.347680+0000", 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031098900000000001 2024-08-17T17:38:44.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352058+0000", 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043780989999999999 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352115+0000", 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7633999999999997e-05 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352160+0000", 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4365e-05 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352284+0000", 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012426200000000001 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352337+0000", 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2898000000000001e-05 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198425 8.13 8:cf361603:::benchmark_data_smithi064_242541_object75932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.484055+0000", 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.580854803000001, 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87490051199999996, 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198425 2024-08-17T17:38:44.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484055+0000", 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484055+0000", 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484057+0000", 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6190000000000001e-06 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484062+0000", 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5380000000000002e-06 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484080+0000", 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7711999999999999e-05 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484089+0000", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0250000000000008e-06 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484672+0000", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000583188 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.484681+0000", 2024-08-17T17:38:44.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6049999999999998e-06 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.350196+0000", 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86551535599999996 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.350295+0000", 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9465999999999999e-05 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.350588+0000", 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029297500000000001 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358705+0000", 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081167410000000002 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358766+0000", 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0846e-05 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358812+0000", 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6103999999999998e-05 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358858+0000", 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5904e-05 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358955+0000", 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7433e-05 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198432 8.13 8:cda9927f:::benchmark_data_smithi064_242541_object75939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.491674+0000", 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.573235339, 2024-08-17T17:38:44.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87608403400000001, 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198432 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.146 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491674+0000", 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491674+0000", 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491676+0000", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3289999999999999e-06 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491680+0000", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6600000000000003e-06 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491694+0000", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3322000000000001e-05 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491701+0000", 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.985e-06 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491759+0000", 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8279000000000002e-05 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.491767+0000", 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8709999999999997e-06 2024-08-17T17:38:44.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352520+0000", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86075362499999997 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352672+0000", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015139600000000001 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.352955+0000", 2024-08-17T17:38:44.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028340700000000001 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.358998+0000", 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060424329999999998 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359036+0000", 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7905999999999999e-05 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367616+0000", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085802529999999995 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367712+0000", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6440999999999994e-05 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367758+0000", 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6127000000000001e-05 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.152 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198462 8.13 8:ce9327bb:::benchmark_data_smithi064_242541_object75969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.525574+0000", 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.539336001999999, 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.842362944, 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198462 2024-08-17T17:38:44.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525574+0000", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525574+0000", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525575+0000", 2024-08-17T17:38:44.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.494e-06 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525581+0000", 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5779999999999998e-06 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525593+0000", 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2330999999999999e-05 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525602+0000", 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6049999999999998e-06 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525661+0000", 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8989000000000001e-05 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.525670+0000", 2024-08-17T17:38:44.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3610000000000007e-06 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.354873+0000", 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.829202683 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.355000+0000", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012710999999999999 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.355286+0000", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000285779 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359083+0000", 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037974369999999999 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359124+0000", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0581000000000003e-05 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367804+0000", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086805660000000007 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367896+0000", 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1955999999999998e-05 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367937+0000", 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0473999999999999e-05 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198496 8.13 8:c8e62d0c:::benchmark_data_smithi064_242541_object76003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.552303+0000", 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.512606466000001, 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81594683899999998, 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.161 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198496 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552303+0000", 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552303+0000", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552304+0000", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5000000000000002e-07 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552308+0000", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9299999999999996e-06 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552318+0000", 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0131e-05 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552325+0000", 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0679999999999999e-06 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552380+0000", 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4664999999999999e-05 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.552390+0000", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7410000000000004e-06 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359154+0000", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80676467399999996 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359263+0000", 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108897 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.359653+0000", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000389894 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367943+0000", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082902900000000005 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368073+0000", 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129357 2024-08-17T17:38:44.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368110+0000", 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7078e-05 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368209+0000", 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9287e-05 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368250+0000", 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1077e-05 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198506 8.13 8:cb27484d:::benchmark_data_smithi064_242541_object76013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.562907+0000", 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.502002625000003, 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80553628099999997, 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.168 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198506 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562907+0000", 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562907+0000", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562908+0000", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9299999999999996e-07 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562912+0000", 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4329999999999997e-06 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562924+0000", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1133e-05 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.562931+0000", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9269999999999999e-06 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.563267+0000", 2024-08-17T17:38:44.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033630699999999999 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.563274+0000", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4619999999999999e-06 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.361887+0000", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79861283500000002 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.362003+0000", 2024-08-17T17:38:44.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115407 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.362288+0000", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028585600000000002 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.367948+0000", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056598910000000002 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368298+0000", 2024-08-17T17:38:44.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034928299999999998 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368344+0000", 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6071e-05 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368396+0000", 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2203000000000001e-05 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368443+0000", 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7580000000000002e-05 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198515 8.13 8:cc3cd848:::benchmark_data_smithi064_242541_object76022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.571784+0000", 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.493126195999999, 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80218924199999997, 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.175 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198515 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571784+0000", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571784+0000", 2024-08-17T17:38:44.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571785+0000", 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.361e-06 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571790+0000", 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1139999999999997e-06 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571803+0000", 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2947e-05 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571809+0000", 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8540000000000002e-06 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571864+0000", 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4741000000000003e-05 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.571874+0000", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0463e-05 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.364318+0000", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79244386200000005 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.365657+0000", 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013388950000000001 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.366003+0000", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034658999999999997 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368456+0000", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002452727 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.373766+0000", 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053099080000000003 2024-08-17T17:38:44.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.373815+0000", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9221999999999997e-05 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.373926+0000", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011102799999999999 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.373973+0000", 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6529999999999997e-05 2024-08-17T17:38:44.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198546 8.13 8:c9465340:::benchmark_data_smithi064_242541_object76053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.626133+0000", 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.438776699000002, 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75401147400000001, 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.182 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198546 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626133+0000", 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626133+0000", 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626135+0000", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9429999999999999e-06 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626142+0000", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626160+0000", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8051e-05 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626166+0000", 2024-08-17T17:38:44.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7299999999999999e-06 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626257+0000", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0734000000000002e-05 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626267+0000", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0088000000000001e-05 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368538+0000", 2024-08-17T17:38:44.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74227086499999995 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.368661+0000", 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012318799999999999 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.369016+0000", 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000354679 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.373778+0000", 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047624929999999996 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380003+0000", 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062251340000000002 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380049+0000", 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5368999999999999e-05 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380102+0000", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3514999999999998e-05 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380145+0000", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2147999999999997e-05 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198547 8.13 8:cce02ce1:::benchmark_data_smithi064_242541_object76054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:11.626301+0000", 2024-08-17T17:38:44.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 32.438608668000001, 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75400550099999997, 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198547 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626301+0000", 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626301+0000", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626302+0000", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3200000000000003e-07 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626306+0000", 2024-08-17T17:38:44.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.464e-06 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626326+0000", 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9930999999999999e-05 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626335+0000", 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2539999999999992e-06 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626392+0000", 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7417000000000003e-05 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.626401+0000", 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5220000000000007e-06 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.370952+0000", 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74455138899999995 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.371056+0000", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010388800000000001 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.371362+0000", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030592100000000001 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.376557+0000", 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051954430000000001 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380185+0000", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003627491 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380218+0000", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3581000000000003e-05 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380266+0000", 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7837999999999998e-05 2024-08-17T17:38:44.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:12.380307+0000", 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0429999999999997e-05 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:38:44.195 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:38:44.196 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:43] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-08-17T17:38:44.220 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:44.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:44.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:44.480 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:44.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:44.582 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:44.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:44.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:44.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.742 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:44.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[104516]: pgmap v1071: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 776/90447 objects misplaced (0.858%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[104516]: 8.1d deep-scrub ok 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[104516]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[109246]: pgmap v1071: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 776/90447 objects misplaced (0.858%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[109246]: 8.1d deep-scrub ok 2024-08-17T17:38:44.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:44 smithi064 ceph-mon[109246]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:38:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:44 smithi175 ceph-mon[101434]: pgmap v1071: 109 pgs: 1 active+remapped+backfilling, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 776/90447 objects misplaced (0.858%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:38:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:44 smithi175 ceph-mon[101434]: 8.1d deep-scrub ok 2024-08-17T17:38:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:44 smithi175 ceph-mon[101434]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:38:44.920 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:38:44.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-08-17T17:38:44.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:45.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:45.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:45.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:45.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:45.255 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:45.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:45.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:45.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.506 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:45.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:45.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:45.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:46.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:46.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:46.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:46.188 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:46.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:46.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:46.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[104516]: pgmap v1074: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:38:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[104516]: 8.20 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11272 2024-08-17T17:38:46.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[104516]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:38:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[109246]: pgmap v1074: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:38:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[109246]: 8.20 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11272 2024-08-17T17:38:46.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:46 smithi064 ceph-mon[109246]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:38:46.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:46.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:38:46.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:46.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:46.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:46 smithi175 ceph-mon[101434]: pgmap v1074: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 105 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:38:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:46 smithi175 ceph-mon[101434]: 8.20 starting backfill to osd.4 from (0'0,0'0] MAX to 339'11272 2024-08-17T17:38:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:46 smithi175 ceph-mon[101434]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:38:46.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.760 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:46.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:46.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:46.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:46.914 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:47.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:47.042 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:47.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:47.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:47.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:47.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:47.423 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:47.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:47.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:47.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:47.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:47.724 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:47.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:47.866 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:47.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:48.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:48.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:48.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.287 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:48.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:48.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:48.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:48.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:48.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:48 smithi175 ceph-mon[101434]: pgmap v1076: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1184/82905 objects misplaced (1.428%) 2024-08-17T17:38:48.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:48 smithi175 ceph-mon[101434]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:38:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:48 smithi064 ceph-mon[104516]: pgmap v1076: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1184/82905 objects misplaced (1.428%) 2024-08-17T17:38:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:48 smithi064 ceph-mon[104516]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:38:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:48 smithi064 ceph-mon[109246]: pgmap v1076: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1184/82905 objects misplaced (1.428%) 2024-08-17T17:38:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:48 smithi064 ceph-mon[109246]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:38:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:48.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:48.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:48.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:48.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:49.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:49.068 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:49.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:49.191 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:49.292 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:49.313 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:49.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:49 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"} : dispatch 2024-08-17T17:38:49.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:49 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"} : dispatch 2024-08-17T17:38:49.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:49.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:49.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:49.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:49.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:49 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"} : dispatch 2024-08-17T17:38:49.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:49.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:49.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:49.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.816 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:49.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:49.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:49.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:49.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:50.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:50.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:50.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:50.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:50.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:50.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:50.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:50.457 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:50.486 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:50 smithi175 ceph-mon[101434]: pgmap v1078: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1184/80595 objects misplaced (1.469%) 2024-08-17T17:38:50.486 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"}]': finished 2024-08-17T17:38:50.486 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:50 smithi175 ceph-mon[101434]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:38:50.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:50.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[104516]: pgmap v1078: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1184/80595 objects misplaced (1.469%) 2024-08-17T17:38:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"}]': finished 2024-08-17T17:38:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[104516]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:38:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[109246]: pgmap v1078: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 1184/80595 objects misplaced (1.469%) 2024-08-17T17:38:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "34"}]': finished 2024-08-17T17:38:50.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:50 smithi064 ceph-mon[109246]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:38:50.693 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:50.722 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:50.776 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:38:50.776 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:38:50.776 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:38:50.776 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:38:50.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:50.851 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:50.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:50.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:51.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:51.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:51.169 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:51.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.235 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:51.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:51.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:51.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:51.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:51 smithi175 ceph-mon[101434]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:38:51.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:51 smithi064 ceph-mon[104516]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:38:51.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:51 smithi064 ceph-mon[109246]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:38:51.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.504 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:51.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:51.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:38:51.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:51.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:51.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:51.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:51.944 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:52.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:52.025 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:52.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:52.159 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:52.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:52.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:52.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:52.429 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:52.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:52.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:52.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:52 smithi175 ceph-mon[101434]: pgmap v1081: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 14 MiB/s, 223 objects/s recovering 2024-08-17T17:38:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:52 smithi175 ceph-mon[101434]: 8.21 starting backfill to osd.5 from (0'0,0'0] MAX to 343'13841 2024-08-17T17:38:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:52 smithi175 ceph-mon[101434]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:38:52.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[104516]: pgmap v1081: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 14 MiB/s, 223 objects/s recovering 2024-08-17T17:38:52.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[104516]: 8.21 starting backfill to osd.5 from (0'0,0'0] MAX to 343'13841 2024-08-17T17:38:52.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[104516]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:38:52.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[109246]: pgmap v1081: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 14 MiB/s, 223 objects/s recovering 2024-08-17T17:38:52.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[109246]: 8.21 starting backfill to osd.5 from (0'0,0'0] MAX to 343'13841 2024-08-17T17:38:52.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:52 smithi064 ceph-mon[109246]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:38:52.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:52.720 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:52.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:52.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:52.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:52.997 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.099 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:53.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:53.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:53.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:53.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:53.580 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:53.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:53.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:53.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:53.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:53.949 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:38:53.949 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:38:53.950 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:38:53.950 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:38:53.950 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:38:53.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:54.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:54.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:54.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:54.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:38:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:38:53] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.51.0" 2024-08-17T17:38:54.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:54.373 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:38:54.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:54.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:54.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:54.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:54 smithi175 ceph-mon[101434]: pgmap v1083: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:38:54.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:54.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:54 smithi064 ceph-mon[104516]: pgmap v1083: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:38:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:54 smithi064 ceph-mon[109246]: pgmap v1083: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:38:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:54.706 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:54.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:54.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:54.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:54.953 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:54.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:55.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.091 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:55.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:55.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:55.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:55.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:55.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:55.355 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:55.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:55.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:55.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:55 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:55 smithi175 ceph-mon[101434]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:38:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:55.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:55.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[109246]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3391331465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[104516]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:38:55.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:55.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:55.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:55.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:55.804 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:38:55.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:55.889 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:38:55.890 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:55.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:55.925 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:38:55.925 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:55.949 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:38:55.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:55.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:38:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:56.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:38:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:56.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:56.030 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:38:56.030 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:56.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:56.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:38:56.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:56.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:38:56.091 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:56.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:56.172 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:56.177 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:56.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:56.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:56.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:56.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:56.489 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:56 smithi175 ceph-mon[101434]: pgmap v1085: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:38:56.489 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:56 smithi175 ceph-mon[101434]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:38:56.489 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:56 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1390794859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:56.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:56.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:56.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[104516]: pgmap v1085: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[104516]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1390794859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[109246]: pgmap v1085: 109 pgs: 1 unknown, 108 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[109246]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:38:56.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:56 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1390794859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:56.714 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:56.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.853 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:56.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:56.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:56.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:57.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:57.118 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:57.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:57.197 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.78938 -> 3.78938 (expected baseline 5.98044) 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 33 -> 33 pgs (0.807339 -> 0.807339 * mean) 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.4 with 46 -> 46 pgs (1.12538 -> 1.12538 * mean) 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0122 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.1223 2024-08-17T17:38:57.198 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2024-08-17T17:38:57.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:57.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:57.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:57.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:57.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:57.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:57.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:57.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:57.653 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:57 smithi175 ceph-mon[101434]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:38:57.653 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"} : dispatch 2024-08-17T17:38:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:57 smithi064 ceph-mon[104516]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:38:57.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"} : dispatch 2024-08-17T17:38:57.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:57 smithi064 ceph-mon[109246]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:38:57.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"} : dispatch 2024-08-17T17:38:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:57.790 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:57.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:57.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:57.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:57.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:58.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:58.097 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:58.121 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-17T17:38:58.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-08-17T17:38:58.124 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:58.124 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:58.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:58.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:58.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:58.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:58.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:58.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:58.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:58.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:58.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:58 smithi175 ceph-mon[101434]: from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:58 smithi175 ceph-mon[101434]: pgmap v1088: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:38:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"}]': finished 2024-08-17T17:38:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:58 smithi175 ceph-mon[101434]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:38:58.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:58 smithi175 ceph-mon[101434]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:38:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[104516]: from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[104516]: pgmap v1088: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:38:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"}]': finished 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[104516]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[104516]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[109246]: from='client.45710 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[109246]: pgmap v1088: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "35"}]': finished 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[109246]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:38:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:58 smithi064 ceph-mon[109246]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:38:58.743 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:58.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:58.865 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:58.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:58.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:38:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:59.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:59.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:59.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:59.124 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:59.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:59.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:59.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:59.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:59.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:59.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:59.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:38:59.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:59.525 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:59.583 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"} : dispatch 2024-08-17T17:38:59.583 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"}]': finished 2024-08-17T17:38:59.583 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:38:59 smithi175 ceph-mon[101434]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:38:59.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:59.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:38:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"} : dispatch 2024-08-17T17:38:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"}]': finished 2024-08-17T17:38:59.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[104516]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:38:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"} : dispatch 2024-08-17T17:38:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "36"}]': finished 2024-08-17T17:38:59.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:38:59 smithi064 ceph-mon[109246]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:38:59.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:59.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.936 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:38:59.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:00.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:00.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:00.208 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:00.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:00.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:00.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:00.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:00.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:00.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:00.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:00 smithi175 ceph-mon[101434]: pgmap v1091: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:39:00.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:00 smithi175 ceph-mon[101434]: 8.22 starting backfill to osd.1 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:00 smithi175 ceph-mon[101434]: 8.22 starting backfill to osd.3 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:00 smithi175 ceph-mon[101434]: 8.22 starting backfill to osd.7 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.645 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:00 smithi175 ceph-mon[101434]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:39:00.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[104516]: pgmap v1091: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:39:00.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[104516]: 8.22 starting backfill to osd.1 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[104516]: 8.22 starting backfill to osd.3 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[104516]: 8.22 starting backfill to osd.7 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[104516]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:39:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[109246]: pgmap v1091: 109 pgs: 1 active+clean+remapped, 108 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:39:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[109246]: 8.22 starting backfill to osd.1 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[109246]: 8.22 starting backfill to osd.3 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[109246]: 8.22 starting backfill to osd.7 from (0'0,0'0] MAX to 349'11681 2024-08-17T17:39:00.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:00 smithi064 ceph-mon[109246]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:39:00.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:00.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:00.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:00.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:00.902 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:00.902 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:00.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:00.995 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:01.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:01.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:01.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:01.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.505 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:01.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:01.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:39:01.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:01.646 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:01.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:01.779 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:01.881 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:01.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:01.932 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:02.019 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:02.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:02.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:02.171 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:02.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:02.296 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:02.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:02.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:02.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:02.578 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:02.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout:moved 5 / 327 (1.52905%) 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 4.10602 -> 4.45639 (expected baseline 5.98044) 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 33 -> 32 pgs (0.807339 -> 0.782875 * mean) 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.4 with 45 -> 46 pgs (1.10092 -> 1.12538 * mean) 2024-08-17T17:39:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0105 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0115 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 weight 1.0000 -> 0.9500 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 1.0000 -> 0.9500 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 1.0000 -> 0.9500 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 1.0000 -> 0.9500 2024-08-17T17:39:02.652 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[104516]: pgmap v1094: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 887 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 444/42462 objects misplaced (1.046%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[104516]: 8.23 starting backfill to osd.0 from (0'0,0'0] MAX to 351'11451 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[104516]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[109246]: pgmap v1094: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 887 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 444/42462 objects misplaced (1.046%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[109246]: 8.23 starting backfill to osd.0 from (0'0,0'0] MAX to 351'11451 2024-08-17T17:39:02.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-mon[109246]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:39:02.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:02 smithi175 ceph-mon[101434]: pgmap v1094: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 887 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 444/42462 objects misplaced (1.046%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:02 smithi175 ceph-mon[101434]: 8.23 starting backfill to osd.0 from (0'0,0'0] MAX to 351'11451 2024-08-17T17:39:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:02 smithi175 ceph-mon[101434]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:39:02.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:02.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:02.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:02.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.935 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:39:02.649+0000 7f06693d0640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-08-17T17:39:02.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:02.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:03.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:03.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:03.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:03.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:03.337 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:03.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:03.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:03.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:03 smithi175 ceph-mon[101434]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:39:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:03 smithi175 ceph-mon[101434]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:03.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:03.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:03 smithi064 ceph-mon[104516]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:39:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:03 smithi064 ceph-mon[104516]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:03.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:03 smithi064 ceph-mon[109246]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:39:03.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:03 smithi064 ceph-mon[109246]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:03.571 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:03.573 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-17T17:39:03.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-17T17:39:03.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:03.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:03.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:03.775 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:03.775 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:03.804 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:03.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:03.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:04.066 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.107 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:04.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:03] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-08-17T17:39:04.208 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:04.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:04.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:04.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:04 smithi175 ceph-mon[101434]: pgmap v1097: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 829 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 444/39642 objects misplaced (1.120%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:04 smithi175 ceph-mon[101434]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:39:04.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:04.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:39:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.514 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:04.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:04.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:04.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:04 smithi064 ceph-mon[104516]: pgmap v1097: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 829 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 444/39642 objects misplaced (1.120%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:04 smithi064 ceph-mon[104516]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:39:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:04 smithi064 ceph-mon[109246]: pgmap v1097: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 829 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 444/39642 objects misplaced (1.120%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:39:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:04 smithi064 ceph-mon[109246]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:39:04.717 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:04.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:04.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:04.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:04.894 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:04.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:04.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:05.042 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:05.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:05.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:05.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:05.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:05.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:05.356 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:39:05.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3285710797' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:05 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3285710797' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:05 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3285710797' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:05 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:05.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:05.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:05.562 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:05.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:05.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:05.736 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:05.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:05.871 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:05.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:06.010 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:06.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:06.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:06.251 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:06.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:06.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:06.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:06.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:06.506 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:06 smithi175 ceph-mon[101434]: pgmap v1099: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 777 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 444/37149 objects misplaced (1.195%) 2024-08-17T17:39:06.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:06 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:06.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:06 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:06.507 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:06 smithi175 ceph-mon[101434]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:39:06.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:06.564 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:06.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[104516]: pgmap v1099: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 777 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 444/37149 objects misplaced (1.195%) 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[104516]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[109246]: pgmap v1099: 109 pgs: 1 unknown, 1 active+remapped+backfilling, 107 active+clean; 777 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 444/37149 objects misplaced (1.195%) 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:06.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:06 smithi064 ceph-mon[109246]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:39:06.663 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:06.664 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:06.666 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:06.701 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:06.712 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:06.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:06.837 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:06.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:06.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:07.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:07.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:07.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:07.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:07.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:07.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:39:07.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:07.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:07.528 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:07.630 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:07.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:07.767 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:07.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:07.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:07.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:07.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:08.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:08.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:08.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:39:08.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:08.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:08.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:08.383 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:08.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:08.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:08.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:08 smithi064 ceph-mon[104516]: pgmap v1101: 109 pgs: 109 active+clean; 561 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:39:08.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1417316095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:08.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:08 smithi064 ceph-mon[109246]: pgmap v1101: 109 pgs: 109 active+clean; 561 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:39:08.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1417316095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:08.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:08.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:08.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:08.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:08.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:08.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:08 smithi175 ceph-mon[101434]: pgmap v1101: 109 pgs: 109 active+clean; 561 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:39:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1417316095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:08.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:08.800 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:08.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:08.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:09.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.079 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:09.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:09.179 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:09.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:09.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:09.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:09.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:09.645 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:09.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:09.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:09.825 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:09.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:09.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:10.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:10.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:10.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:241354 8.8 8:15e3d8a6:::benchmark_data_smithi064_242541_object118861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:39:10.034278+0000", 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.027106967999999999, 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.027131064999999999, 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:10.190 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 241354 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034278+0000", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:10.191 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034278+0000", 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034278+0000", 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034281+0000", 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4999999999999999e-06 2024-08-17T17:39:10.192 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034287+0000", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1839999999999998e-06 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.034291+0000", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3850000000000004e-06 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.036236+0000", 2024-08-17T17:39:10.193 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001945331 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.036463+0000", 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000226966 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.039197+0000", 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0027332540000000001 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.194 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.048553+0000", 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093568130000000003 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:39:10.054525+0000", 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0059713020000000004 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:10.195 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:39:10.196 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 1 2024-08-17T17:39:10.196 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:39:10.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:10.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:10.345 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:10.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout:moved 5 / 327 (1.52905%) 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 4.10602 -> 4.45639 (expected baseline 5.98044) 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 33 -> 32 pgs (0.807339 -> 0.782875 * mean) 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.4 with 45 -> 46 pgs (1.10092 -> 1.12538 * mean) 2024-08-17T17:39:10.385 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0090 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0099 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 weight 1.0000 -> 0.9500 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 1.0000 -> 0.9500 2024-08-17T17:39:10.386 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 1.0000 -> 0.9500 2024-08-17T17:39:10.447 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:10.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:10.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:10.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:10.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:10 smithi175 ceph-mon[101434]: pgmap v1102: 109 pgs: 109 active+clean; 500 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:39:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"} : dispatch 2024-08-17T17:39:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[104516]: pgmap v1102: 109 pgs: 109 active+clean; 500 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:39:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"} : dispatch 2024-08-17T17:39:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[109246]: pgmap v1102: 109 pgs: 109 active+clean; 500 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:39:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:10.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"} : dispatch 2024-08-17T17:39:10.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:10.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:10.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:10.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:10.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:10.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:11.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.113 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:11.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:11.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:11.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:11.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.362 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:11.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:11.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:11.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:11.485 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:11.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:11.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:11.610 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:11 smithi175 ceph-mon[101434]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"}]': finished 2024-08-17T17:39:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:11 smithi175 ceph-mon[101434]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:39:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[104516]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"}]': finished 2024-08-17T17:39:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[104516]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:39:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[109246]: from='client.45728 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:39:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','7':'62260'}"}]': finished 2024-08-17T17:39:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:11 smithi064 ceph-mon[109246]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:39:11.712 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.738 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:11.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:11.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:11.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:11.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:39:11.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:11.985 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:12.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:12.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:12.186 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:12.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:12.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:12.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:12.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:12.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[104516]: pgmap v1104: 109 pgs: 1 peering, 108 active+clean; 285 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:12.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:39:12.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:12.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:12.549 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:12 smithi175 ceph-mon[101434]: pgmap v1104: 109 pgs: 1 peering, 108 active+clean; 285 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:12.549 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:12 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:39:12.549 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:12 smithi175 ceph-mon[101434]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:39:12.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:12.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[104516]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:39:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[109246]: pgmap v1104: 109 pgs: 1 peering, 108 active+clean; 285 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:39:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:12 smithi064 ceph-mon[109246]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:39:12.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:12.747 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:12.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:12.889 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:12.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:13.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:13.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:13.157 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:13.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:13.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:13.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.459 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:13.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:13.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:13 smithi175 ceph-mon[101434]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:39:13.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:13 smithi175 ceph-mon[101434]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:13 smithi175 ceph-mon[101434]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.594 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:13.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[104516]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[104516]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[104516]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[109246]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[109246]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-mon[109246]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 357'13393 2024-08-17T17:39:13.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:13.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:13.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:13.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:13.992 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:14.014 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:39:14.014 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:39:14.014 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:39:14.014 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:39:14.015 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:39:14.044 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:14.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:14.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:14.208 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:13] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-08-17T17:39:14.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:14.269 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:14.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:14.404 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:14.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:14.461 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:14 smithi064 ceph-mon[104516]: pgmap v1107: 109 pgs: 2 peering, 107 active+clean; 209 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:39:14.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:14 smithi064 ceph-mon[109246]: pgmap v1107: 109 pgs: 2 peering, 107 active+clean; 209 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:39:14.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:14.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:14.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:14 smithi175 ceph-mon[101434]: pgmap v1107: 109 pgs: 2 peering, 107 active+clean; 209 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:39:14.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:14.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:14.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:14.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:14.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:14.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:15.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:15.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:15.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:15.171 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:15.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:15.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:15.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:15.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:15.539 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 122490 objects 2024-08-17T17:39:15.539 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :162.354 2024-08-17T17:39:15.548 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:15.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:39:15.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:15.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:15.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:15 smithi064 ceph-mon[104516]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:39:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:15 smithi064 ceph-mon[109246]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:39:15.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:15.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:15.829 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:15.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:15 smithi175 ceph-mon[101434]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:39:15.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:15.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:16.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:16.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:16.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:16.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:16.307 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:16.401 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:16.402 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:16.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:16.418 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:16.418 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:16.426 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:39:16.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:16.437 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:16.437 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:16.455 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:16.455 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:16.474 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:16.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:16.490 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:16.490 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:16.510 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:16.510 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:16.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:16.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:16.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:16.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:16.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:16.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[104516]: pgmap v1109: 109 pgs: 1 active+clean+remapped, 2 peering, 106 active+clean; 157 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1008 KiB/s, 15 objects/s recovering 2024-08-17T17:39:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[104516]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:39:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:16.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2113875910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[109246]: pgmap v1109: 109 pgs: 1 active+clean+remapped, 2 peering, 106 active+clean; 157 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1008 KiB/s, 15 objects/s recovering 2024-08-17T17:39:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[109246]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:39:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:16.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2113875910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:16.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:16.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:16 smithi175 ceph-mon[101434]: pgmap v1109: 109 pgs: 1 active+clean+remapped, 2 peering, 106 active+clean; 157 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1008 KiB/s, 15 objects/s recovering 2024-08-17T17:39:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:16 smithi175 ceph-mon[101434]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:39:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:16.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2113875910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:17.034 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:17.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:17.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:17.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:17.286 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:17.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:17.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:17.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:17.539 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:17.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:17.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:17.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[104516]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:39:17.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:17.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:17.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:17 smithi064 ceph-mon[109246]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:39:17.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:17.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:17.689 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:17.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:17 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:17.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:17 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2571684259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:17.739 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:17 smithi175 ceph-mon[101434]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:39:17.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:17.815 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:17.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:18.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:18.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:18.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:18.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:18.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:18.444 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:18.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:18.468 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:18 smithi175 ceph-mon[101434]: pgmap v1112: 109 pgs: 1 active+clean+remapped, 108 active+clean; 22 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1010 KiB/s, 17 objects/s recovering 2024-08-17T17:39:18.468 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:18 smithi175 ceph-mon[101434]: 8.10 scrub starts 2024-08-17T17:39:18.468 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:18 smithi175 ceph-mon[101434]: 8.10 scrub ok 2024-08-17T17:39:18.468 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:18 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:39:18.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:18.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:18.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[104516]: pgmap v1112: 109 pgs: 1 active+clean+remapped, 108 active+clean; 22 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1010 KiB/s, 17 objects/s recovering 2024-08-17T17:39:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[104516]: 8.10 scrub starts 2024-08-17T17:39:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[104516]: 8.10 scrub ok 2024-08-17T17:39:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:39:18.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[109246]: pgmap v1112: 109 pgs: 1 active+clean+remapped, 108 active+clean; 22 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1010 KiB/s, 17 objects/s recovering 2024-08-17T17:39:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[109246]: 8.10 scrub starts 2024-08-17T17:39:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[109246]: 8.10 scrub ok 2024-08-17T17:39:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:18 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:39:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:18.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:18.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:18.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:18.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:18.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:18.950 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.997 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:18.997 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:19.038 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:39:19.038 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:39:19.039 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-17T17:39:19.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:19.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:19.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:19.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:19.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:19.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:19.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:19.270 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:19.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:19.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.357 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:19.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:19.461 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:19.482 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:19.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:19.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:19.608 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:39:19.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:19.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:39:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.731 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:19.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:19.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:19.850 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:19.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:19.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:19.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:20.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:20.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:20.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:20.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:20.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:20.350 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:20.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:20.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:20.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:20.597 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:20.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:20 smithi175 ceph-mon[101434]: pgmap v1113: 109 pgs: 109 active+clean; 7.2 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 811 KiB/s, 13 objects/s recovering 2024-08-17T17:39:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:20 smithi175 ceph-mon[101434]: 8.15 scrub starts 2024-08-17T17:39:20.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:20 smithi175 ceph-mon[101434]: 8.15 scrub ok 2024-08-17T17:39:20.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[104516]: pgmap v1113: 109 pgs: 109 active+clean; 7.2 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 811 KiB/s, 13 objects/s recovering 2024-08-17T17:39:20.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[104516]: 8.15 scrub starts 2024-08-17T17:39:20.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[104516]: 8.15 scrub ok 2024-08-17T17:39:20.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[109246]: pgmap v1113: 109 pgs: 109 active+clean; 7.2 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 811 KiB/s, 13 objects/s recovering 2024-08-17T17:39:20.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[109246]: 8.15 scrub starts 2024-08-17T17:39:20.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:20 smithi064 ceph-mon[109246]: 8.15 scrub ok 2024-08-17T17:39:20.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:20.721 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:20.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:20.764 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:20.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:20.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:20.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:20.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:21.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:21.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:21.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:21.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:21.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.296 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:21.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:21.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:21.440 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:21.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:21.462 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:21.462 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:21.482 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:39:21.483 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:39:21.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:21.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:21.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:21.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:21.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:21.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:21.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:21.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:21.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:22.055 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:39:22.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:22.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:22.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:22.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:22.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:22.260 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:22.302 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1114,"stamp":"2024-08-17T17:39:21.211414+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5085157,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":6378,"num_read_kb":13833,"num_write":61417,"num_write_kb":505697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14601,"num_bytes_recovered":958996291,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":371919,"ondisk_log_size":371919,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4872616,"kb_used_data":56824,"kb_used_omap":46,"kb_used_meta":4815633,"kb_avail":745088600,"statfs":{"total":767960285184,"available":762970726400,"internally_reserved":0,"allocated":58187776,"data_stored":43723406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":4931208435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-167051288,"num_objects":-2550,"num_object_clones":0,"num_object_copies":-7650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2637,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.794794"},"pg_stats":[{"pgid":"8.23","version":"362'11686","reported_seq":12395,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845523+0000","last_change":"2024-08-17T17:39:03.356479+0000","last_active":"2024-08-17T17:39:16.845523+0000","last_peered":"2024-08-17T17:39:16.845523+0000","last_clean":"2024-08-17T17:39:16.845523+0000","last_became_active":"2024-08-17T17:39:03.339890+0000","last_became_peered":"2024-08-17T17:39:03.339890+0000","last_unstale":"2024-08-17T17:39:16.845523+0000","last_undegraded":"2024-08-17T17:39:16.845523+0000","last_fullsized":"2024-08-17T17:39:16.845523+0000","mapping_epoch":355,"log_start":"279'2400","ondisk_log_start":"279'2400","created":304,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":6,"last_scrub":"312'10946","last_scrub_stamp":"2024-08-17T17:38:09.720944+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:09.720944+0000","objects_scrubbed":740,"log_size":9286,"log_dups_size":0,"ondisk_log_size":9286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:41:35.046712+0000","scrub_duration":115,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.22","version":"362'11919","reported_seq":12989,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.479862+0000","last_change":"2024-08-17T17:39:02.349962+0000","last_active":"2024-08-17T17:39:15.479862+0000","last_peered":"2024-08-17T17:39:15.479862+0000","last_clean":"2024-08-17T17:39:15.479862+0000","last_became_active":"2024-08-17T17:39:02.334537+0000","last_became_peered":"2024-08-17T17:39:02.334537+0000","last_unstale":"2024-08-17T17:39:15.479862+0000","last_undegraded":"2024-08-17T17:39:15.479862+0000","last_fullsized":"2024-08-17T17:39:15.479862+0000","mapping_epoch":354,"log_start":"279'2200","ondisk_log_start":"279'2200","created":304,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":6,"last_scrub":"325'11295","last_scrub_stamp":"2024-08-17T17:38:25.397429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:25.397429+0000","objects_scrubbed":624,"log_size":9719,"log_dups_size":0,"ondisk_log_size":9719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:21.156628+0000","scrub_duration":0.099256820999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":624,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":13828096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.21","version":"361'14172","reported_seq":15092,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845728+0000","last_change":"2024-08-17T17:38:56.400425+0000","last_active":"2024-08-17T17:39:16.845728+0000","last_peered":"2024-08-17T17:39:16.845728+0000","last_clean":"2024-08-17T17:39:16.845728+0000","last_became_active":"2024-08-17T17:38:56.384809+0000","last_became_peered":"2024-08-17T17:38:56.384809+0000","last_unstale":"2024-08-17T17:39:16.845728+0000","last_undegraded":"2024-08-17T17:39:16.845728+0000","last_fullsized":"2024-08-17T17:39:16.845728+0000","mapping_epoch":348,"log_start":"279'2600","ondisk_log_start":"279'2600","created":304,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":6,"last_scrub":"309'13399","last_scrub_stamp":"2024-08-17T17:38:03.853928+0000","last_deep_scrub":"296'12864","last_deep_scrub_stamp":"2024-08-17T17:37:37.596623+0000","last_clean_scrub_stamp":"2024-08-17T17:38:03.853928+0000","objects_scrubbed":774,"log_size":11572,"log_dups_size":0,"ondisk_log_size":11572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:15.977090+0000","scrub_duration":208,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":774,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":282,"num_bytes_recovered":18481152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.20","version":"361'11701","reported_seq":12756,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422615+0000","last_change":"2024-08-17T17:38:50.361103+0000","last_active":"2024-08-17T17:39:15.422615+0000","last_peered":"2024-08-17T17:39:15.422615+0000","last_clean":"2024-08-17T17:39:15.422615+0000","last_became_active":"2024-08-17T17:38:50.351272+0000","last_became_peered":"2024-08-17T17:38:50.351272+0000","last_unstale":"2024-08-17T17:39:15.422615+0000","last_undegraded":"2024-08-17T17:39:15.422615+0000","last_fullsized":"2024-08-17T17:39:15.422615+0000","mapping_epoch":344,"log_start":"278'2000","ondisk_log_start":"278'2000","created":304,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":6,"last_scrub":"309'10965","last_scrub_stamp":"2024-08-17T17:38:05.214853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:05.214853+0000","objects_scrubbed":736,"log_size":9701,"log_dups_size":0,"ondisk_log_size":9701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:11.960957+0000","scrub_duration":178,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":736,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":392,"num_bytes_recovered":25690112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.18","version":"362'12882","reported_seq":15886,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.532402+0000","last_change":"2024-08-17T17:38:35.792339+0000","last_active":"2024-08-17T17:39:15.532402+0000","last_peered":"2024-08-17T17:39:15.532402+0000","last_clean":"2024-08-17T17:39:15.532402+0000","last_became_active":"2024-08-17T17:38:25.316340+0000","last_became_peered":"2024-08-17T17:38:25.316340+0000","last_unstale":"2024-08-17T17:39:15.532402+0000","last_undegraded":"2024-08-17T17:39:15.532402+0000","last_fullsized":"2024-08-17T17:39:15.532402+0000","mapping_epoch":326,"log_start":"278'2132","ondisk_log_start":"278'2132","created":289,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":5,"last_scrub":"335'11813","last_scrub_stamp":"2024-08-17T17:38:35.792139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:35.792139+0000","objects_scrubbed":1075,"log_size":10750,"log_dups_size":0,"ondisk_log_size":10750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:57.495631+0000","scrub_duration":0.51146411000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1075,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":585,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846231+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:39:16.846231+0000","last_peered":"2024-08-17T17:39:16.846231+0000","last_clean":"2024-08-17T17:39:16.846231+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:39:16.846231+0000","last_undegraded":"2024-08-17T17:39:16.846231+0000","last_fullsized":"2024-08-17T17:39:16.846231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.418262+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:39:14.418262+0000","last_peered":"2024-08-17T17:39:14.418262+0000","last_clean":"2024-08-17T17:39:14.418262+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:39:14.418262+0000","last_undegraded":"2024-08-17T17:39:14.418262+0000","last_fullsized":"2024-08-17T17:39:14.418262+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.19","version":"362'12713","reported_seq":15535,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.538754+0000","last_change":"2024-08-17T17:38:24.339213+0000","last_active":"2024-08-17T17:39:15.538754+0000","last_peered":"2024-08-17T17:39:15.538754+0000","last_clean":"2024-08-17T17:39:15.538754+0000","last_became_active":"2024-08-17T17:38:24.319199+0000","last_became_peered":"2024-08-17T17:38:24.319199+0000","last_unstale":"2024-08-17T17:39:15.538754+0000","last_undegraded":"2024-08-17T17:39:15.538754+0000","last_fullsized":"2024-08-17T17:39:15.538754+0000","mapping_epoch":325,"log_start":"278'913","ondisk_log_start":"278'913","created":289,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":5,"last_scrub":"312'11185","last_scrub_stamp":"2024-08-17T17:38:10.263756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:10.263756+0000","objects_scrubbed":1528,"log_size":11800,"log_dups_size":0,"ondisk_log_size":11800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:17.288710+0000","scrub_duration":0.17575571300000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1528,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1324,"num_bytes_recovered":86769664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":146,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422668+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:39:15.422668+0000","last_peered":"2024-08-17T17:39:15.422668+0000","last_clean":"2024-08-17T17:39:15.422668+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:39:15.422668+0000","last_undegraded":"2024-08-17T17:39:15.422668+0000","last_fullsized":"2024-08-17T17:39:15.422668+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":577,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845707+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:39:16.845707+0000","last_peered":"2024-08-17T17:39:16.845707+0000","last_clean":"2024-08-17T17:39:16.845707+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:39:16.845707+0000","last_undegraded":"2024-08-17T17:39:16.845707+0000","last_fullsized":"2024-08-17T17:39:16.845707+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.1a","version":"362'13472","reported_seq":18848,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845524+0000","last_change":"2024-08-17T17:38:28.356101+0000","last_active":"2024-08-17T17:39:16.845524+0000","last_peered":"2024-08-17T17:39:16.845524+0000","last_clean":"2024-08-17T17:39:16.845524+0000","last_became_active":"2024-08-17T17:38:28.328643+0000","last_became_peered":"2024-08-17T17:38:28.328643+0000","last_unstale":"2024-08-17T17:39:16.845524+0000","last_undegraded":"2024-08-17T17:39:16.845524+0000","last_fullsized":"2024-08-17T17:39:16.845524+0000","mapping_epoch":329,"log_start":"280'4526","ondisk_log_start":"280'4526","created":304,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'11982","last_scrub_stamp":"2024-08-17T17:38:08.789025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:08.789025+0000","objects_scrubbed":1490,"log_size":8946,"log_dups_size":0,"ondisk_log_size":8946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:45.121148+0000","scrub_duration":151,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1490,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1279,"num_bytes_recovered":83820544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":577,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846364+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:39:16.846364+0000","last_peered":"2024-08-17T17:39:16.846364+0000","last_clean":"2024-08-17T17:39:16.846364+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:39:16.846364+0000","last_undegraded":"2024-08-17T17:39:16.846364+0000","last_fullsized":"2024-08-17T17:39:16.846364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.418389+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:39:14.418389+0000","last_peered":"2024-08-17T17:39:14.418389+0000","last_clean":"2024-08-17T17:39:14.418389+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:39:14.418389+0000","last_undegraded":"2024-08-17T17:39:14.418389+0000","last_fullsized":"2024-08-17T17:39:14.418389+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1b","version":"362'13609","reported_seq":17381,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.489355+0000","last_change":"2024-08-17T17:38:33.336548+0000","last_active":"2024-08-17T17:39:15.489355+0000","last_peered":"2024-08-17T17:39:15.489355+0000","last_clean":"2024-08-17T17:39:15.489355+0000","last_became_active":"2024-08-17T17:38:33.320350+0000","last_became_peered":"2024-08-17T17:38:33.320350+0000","last_unstale":"2024-08-17T17:39:15.489355+0000","last_undegraded":"2024-08-17T17:39:15.489355+0000","last_fullsized":"2024-08-17T17:39:15.489355+0000","mapping_epoch":332,"log_start":"280'4368","ondisk_log_start":"280'4368","created":304,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"307'11947","last_scrub_stamp":"2024-08-17T17:38:03.211910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:03.211910+0000","objects_scrubbed":1662,"log_size":9241,"log_dups_size":0,"ondisk_log_size":9241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:27.119035+0000","scrub_duration":225,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1662,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1185,"num_bytes_recovered":77660160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":585,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846669+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:39:16.846669+0000","last_peered":"2024-08-17T17:39:16.846669+0000","last_clean":"2024-08-17T17:39:16.846669+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:39:16.846669+0000","last_undegraded":"2024-08-17T17:39:16.846669+0000","last_fullsized":"2024-08-17T17:39:16.846669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":296,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.469764+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:39:15.469764+0000","last_peered":"2024-08-17T17:39:15.469764+0000","last_clean":"2024-08-17T17:39:15.469764+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:39:15.469764+0000","last_undegraded":"2024-08-17T17:39:15.469764+0000","last_fullsized":"2024-08-17T17:39:15.469764+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"362'12793","reported_seq":12945,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.532953+0000","last_change":"2024-08-17T17:38:37.284021+0000","last_active":"2024-08-17T17:39:15.532953+0000","last_peered":"2024-08-17T17:39:15.532953+0000","last_clean":"2024-08-17T17:39:15.532953+0000","last_became_active":"2024-08-17T17:37:24.862536+0000","last_became_peered":"2024-08-17T17:37:24.862536+0000","last_unstale":"2024-08-17T17:39:15.532953+0000","last_undegraded":"2024-08-17T17:39:15.532953+0000","last_fullsized":"2024-08-17T17:39:15.532953+0000","mapping_epoch":274,"log_start":"278'744","ondisk_log_start":"278'744","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"335'11804","last_scrub_stamp":"2024-08-17T17:38:37.283805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:37.283805+0000","objects_scrubbed":990,"log_size":12049,"log_dups_size":0,"ondisk_log_size":12049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:01:40.285737+0000","scrub_duration":0.21214570399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":990,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.419023+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:39:14.419023+0000","last_peered":"2024-08-17T17:39:14.419023+0000","last_clean":"2024-08-17T17:39:14.419023+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:39:14.419023+0000","last_undegraded":"2024-08-17T17:39:14.419023+0000","last_fullsized":"2024-08-17T17:39:14.419023+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":156,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422169+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:39:15.422169+0000","last_peered":"2024-08-17T17:39:15.422169+0000","last_clean":"2024-08-17T17:39:15.422169+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:39:15.422169+0000","last_undegraded":"2024-08-17T17:39:15.422169+0000","last_fullsized":"2024-08-17T17:39:15.422169+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"362'13074","reported_seq":13223,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.485685+0000","last_change":"2024-08-17T17:38:13.587903+0000","last_active":"2024-08-17T17:39:15.485685+0000","last_peered":"2024-08-17T17:39:15.485685+0000","last_clean":"2024-08-17T17:39:15.485685+0000","last_became_active":"2024-08-17T17:37:24.862433+0000","last_became_peered":"2024-08-17T17:37:24.862433+0000","last_unstale":"2024-08-17T17:39:15.485685+0000","last_undegraded":"2024-08-17T17:39:15.485685+0000","last_fullsized":"2024-08-17T17:39:15.485685+0000","mapping_epoch":274,"log_start":"278'1100","ondisk_log_start":"278'1100","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"316'11624","last_scrub_stamp":"2024-08-17T17:38:13.587783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:13.587783+0000","objects_scrubbed":1454,"log_size":11974,"log_dups_size":0,"ondisk_log_size":11974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:04.316025+0000","scrub_duration":0.23660852299999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1454,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":357,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.419388+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:39:14.419388+0000","last_peered":"2024-08-17T17:39:14.419388+0000","last_clean":"2024-08-17T17:39:14.419388+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:39:14.419388+0000","last_undegraded":"2024-08-17T17:39:14.419388+0000","last_fullsized":"2024-08-17T17:39:14.419388+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":253,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421656+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:39:15.421656+0000","last_peered":"2024-08-17T17:39:15.421656+0000","last_clean":"2024-08-17T17:39:15.421656+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:39:15.421656+0000","last_undegraded":"2024-08-17T17:39:15.421656+0000","last_fullsized":"2024-08-17T17:39:15.421656+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"362'12695","reported_seq":12948,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846136+0000","last_change":"2024-08-17T17:37:26.287987+0000","last_active":"2024-08-17T17:39:16.846136+0000","last_peered":"2024-08-17T17:39:16.846136+0000","last_clean":"2024-08-17T17:39:16.846136+0000","last_became_active":"2024-08-17T17:37:24.868093+0000","last_became_peered":"2024-08-17T17:37:24.868093+0000","last_unstale":"2024-08-17T17:39:16.846136+0000","last_undegraded":"2024-08-17T17:39:16.846136+0000","last_fullsized":"2024-08-17T17:39:16.846136+0000","mapping_epoch":274,"log_start":"278'954","ondisk_log_start":"278'954","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10217","last_scrub_stamp":"2024-08-17T17:37:26.287558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:26.287558+0000","objects_scrubbed":2490,"log_size":11741,"log_dups_size":0,"ondisk_log_size":11741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:28.567691+0000","scrub_duration":708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2490,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":374,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.419584+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:39:14.419584+0000","last_peered":"2024-08-17T17:39:14.419584+0000","last_clean":"2024-08-17T17:39:14.419584+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:39:14.419584+0000","last_undegraded":"2024-08-17T17:39:14.419584+0000","last_fullsized":"2024-08-17T17:39:14.419584+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":276,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.470956+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:39:15.470956+0000","last_peered":"2024-08-17T17:39:15.470956+0000","last_clean":"2024-08-17T17:39:15.470956+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:39:15.470956+0000","last_undegraded":"2024-08-17T17:39:15.470956+0000","last_fullsized":"2024-08-17T17:39:15.470956+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"362'12757","reported_seq":13011,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551790+0000","last_change":"2024-08-17T17:37:26.443446+0000","last_active":"2024-08-17T17:39:16.551790+0000","last_peered":"2024-08-17T17:39:16.551790+0000","last_clean":"2024-08-17T17:39:16.551790+0000","last_became_active":"2024-08-17T17:37:24.890776+0000","last_became_peered":"2024-08-17T17:37:24.890776+0000","last_unstale":"2024-08-17T17:39:16.551790+0000","last_undegraded":"2024-08-17T17:39:16.551790+0000","last_fullsized":"2024-08-17T17:39:16.551790+0000","mapping_epoch":274,"log_start":"279'2600","ondisk_log_start":"279'2600","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"290'10372","last_scrub_stamp":"2024-08-17T17:37:26.443046+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:26.443046+0000","objects_scrubbed":2396,"log_size":10157,"log_dups_size":0,"ondisk_log_size":10157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:12.865819+0000","scrub_duration":987,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2396,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":156,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422893+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:39:15.422893+0000","last_peered":"2024-08-17T17:39:15.422893+0000","last_clean":"2024-08-17T17:39:15.422893+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:39:15.422893+0000","last_undegraded":"2024-08-17T17:39:15.422893+0000","last_fullsized":"2024-08-17T17:39:15.422893+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":591,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846089+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:39:16.846089+0000","last_peered":"2024-08-17T17:39:16.846089+0000","last_clean":"2024-08-17T17:39:16.846089+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:39:16.846089+0000","last_undegraded":"2024-08-17T17:39:16.846089+0000","last_fullsized":"2024-08-17T17:39:16.846089+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"362'11667","reported_seq":11947,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845847+0000","last_change":"2024-08-17T17:38:08.205322+0000","last_active":"2024-08-17T17:39:16.845847+0000","last_peered":"2024-08-17T17:39:16.845847+0000","last_clean":"2024-08-17T17:39:16.845847+0000","last_became_active":"2024-08-17T17:37:56.315472+0000","last_became_peered":"2024-08-17T17:37:56.315472+0000","last_unstale":"2024-08-17T17:39:16.845847+0000","last_undegraded":"2024-08-17T17:39:16.845847+0000","last_fullsized":"2024-08-17T17:39:16.845847+0000","mapping_epoch":274,"log_start":"278'2000","ondisk_log_start":"278'2000","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'11003","last_scrub_stamp":"2024-08-17T17:38:08.204865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:08.204865+0000","objects_scrubbed":664,"log_size":9667,"log_dups_size":0,"ondisk_log_size":9667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:39.496003+0000","scrub_duration":104,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":664,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":607,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846561+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:39:16.846561+0000","last_peered":"2024-08-17T17:39:16.846561+0000","last_clean":"2024-08-17T17:39:16.846561+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:39:16.846561+0000","last_undegraded":"2024-08-17T17:39:16.846561+0000","last_fullsized":"2024-08-17T17:39:16.846561+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422205+0000","last_change":"2024-08-17T17:31:36.161882+0000","last_active":"2024-08-17T17:39:15.422205+0000","last_peered":"2024-08-17T17:39:15.422205+0000","last_clean":"2024-08-17T17:39:15.422205+0000","last_became_active":"2024-08-17T17:31:36.161587+0000","last_became_peered":"2024-08-17T17:31:36.161587+0000","last_unstale":"2024-08-17T17:39:15.422205+0000","last_undegraded":"2024-08-17T17:39:15.422205+0000","last_fullsized":"2024-08-17T17:39:15.422205+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:07.218474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"362'14223","reported_seq":14566,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.550921+0000","last_change":"2024-08-17T17:38:04.895949+0000","last_active":"2024-08-17T17:39:16.550921+0000","last_peered":"2024-08-17T17:39:16.550921+0000","last_clean":"2024-08-17T17:39:16.550921+0000","last_became_active":"2024-08-17T17:37:56.352073+0000","last_became_peered":"2024-08-17T17:37:56.352073+0000","last_unstale":"2024-08-17T17:39:16.550921+0000","last_undegraded":"2024-08-17T17:39:16.550921+0000","last_fullsized":"2024-08-17T17:39:16.550921+0000","mapping_epoch":274,"log_start":"279'2600","ondisk_log_start":"279'2600","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"309'13432","last_scrub_stamp":"2024-08-17T17:38:04.895575+0000","last_deep_scrub":"296'12864","last_deep_scrub_stamp":"2024-08-17T17:37:37.596623+0000","last_clean_scrub_stamp":"2024-08-17T17:38:04.895575+0000","objects_scrubbed":795,"log_size":11623,"log_dups_size":0,"ondisk_log_size":11623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:56.971312+0000","scrub_duration":239,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":795,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421721+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:39:15.421721+0000","last_peered":"2024-08-17T17:39:15.421721+0000","last_clean":"2024-08-17T17:39:15.421721+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:39:15.421721+0000","last_undegraded":"2024-08-17T17:39:15.421721+0000","last_fullsized":"2024-08-17T17:39:15.421721+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":253,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422318+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:39:15.422318+0000","last_peered":"2024-08-17T17:39:15.422318+0000","last_clean":"2024-08-17T17:39:15.422318+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:39:15.422318+0000","last_undegraded":"2024-08-17T17:39:15.422318+0000","last_fullsized":"2024-08-17T17:39:15.422318+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"362'11893","reported_seq":12102,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.516464+0000","last_change":"2024-08-17T17:38:10.348874+0000","last_active":"2024-08-17T17:39:15.516464+0000","last_peered":"2024-08-17T17:39:15.516464+0000","last_clean":"2024-08-17T17:39:15.516464+0000","last_became_active":"2024-08-17T17:37:56.329845+0000","last_became_peered":"2024-08-17T17:37:56.329845+0000","last_unstale":"2024-08-17T17:39:15.516464+0000","last_undegraded":"2024-08-17T17:39:15.516464+0000","last_fullsized":"2024-08-17T17:39:15.516464+0000","mapping_epoch":274,"log_start":"279'2200","ondisk_log_start":"279'2200","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"312'11182","last_scrub_stamp":"2024-08-17T17:38:10.348706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:10.348706+0000","objects_scrubbed":711,"log_size":9693,"log_dups_size":0,"ondisk_log_size":9693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:51.834359+0000","scrub_duration":0.089319198000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":711,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":253,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421540+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:39:15.421540+0000","last_peered":"2024-08-17T17:39:15.421540+0000","last_clean":"2024-08-17T17:39:15.421540+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:39:15.421540+0000","last_undegraded":"2024-08-17T17:39:15.421540+0000","last_fullsized":"2024-08-17T17:39:15.421540+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":268,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422619+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:39:15.422619+0000","last_peered":"2024-08-17T17:39:15.422619+0000","last_clean":"2024-08-17T17:39:15.422619+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:39:15.422619+0000","last_undegraded":"2024-08-17T17:39:15.422619+0000","last_fullsized":"2024-08-17T17:39:15.422619+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"362'11677","reported_seq":11958,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551662+0000","last_change":"2024-08-17T17:38:10.651678+0000","last_active":"2024-08-17T17:39:16.551662+0000","last_peered":"2024-08-17T17:39:16.551662+0000","last_clean":"2024-08-17T17:39:16.551662+0000","last_became_active":"2024-08-17T17:37:56.312813+0000","last_became_peered":"2024-08-17T17:37:56.312813+0000","last_unstale":"2024-08-17T17:39:16.551662+0000","last_undegraded":"2024-08-17T17:39:16.551662+0000","last_fullsized":"2024-08-17T17:39:16.551662+0000","mapping_epoch":274,"log_start":"279'2400","ondisk_log_start":"279'2400","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"313'10952","last_scrub_stamp":"2024-08-17T17:38:10.651538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:10.651538+0000","objects_scrubbed":725,"log_size":9277,"log_dups_size":0,"ondisk_log_size":9277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:57:33.965857+0000","scrub_duration":79,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":725,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":587,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846338+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:39:16.846338+0000","last_peered":"2024-08-17T17:39:16.846338+0000","last_clean":"2024-08-17T17:39:16.846338+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:39:16.846338+0000","last_undegraded":"2024-08-17T17:39:16.846338+0000","last_fullsized":"2024-08-17T17:39:16.846338+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":573,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.464384+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:39:15.464384+0000","last_peered":"2024-08-17T17:39:15.464384+0000","last_clean":"2024-08-17T17:39:15.464384+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:39:15.464384+0000","last_undegraded":"2024-08-17T17:39:15.464384+0000","last_fullsized":"2024-08-17T17:39:15.464384+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"362'13587","reported_seq":13788,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.532669+0000","last_change":"2024-08-17T17:38:41.951635+0000","last_active":"2024-08-17T17:39:15.532669+0000","last_peered":"2024-08-17T17:39:15.532669+0000","last_clean":"2024-08-17T17:39:15.532669+0000","last_became_active":"2024-08-17T17:37:24.907010+0000","last_became_peered":"2024-08-17T17:37:24.907010+0000","last_unstale":"2024-08-17T17:39:15.532669+0000","last_undegraded":"2024-08-17T17:39:15.532669+0000","last_fullsized":"2024-08-17T17:39:15.532669+0000","mapping_epoch":274,"log_start":"279'2372","ondisk_log_start":"279'2372","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"338'12690","last_scrub_stamp":"2024-08-17T17:38:41.951385+0000","last_deep_scrub":"338'12690","last_deep_scrub_stamp":"2024-08-17T17:38:41.951385+0000","last_clean_scrub_stamp":"2024-08-17T17:38:41.951385+0000","objects_scrubbed":957,"log_size":11215,"log_dups_size":0,"ondisk_log_size":11215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":6,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:59.387728+0000","scrub_duration":5.9059909929999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":957,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":677,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.418266+0000","last_change":"2024-08-17T17:31:36.156846+0000","last_active":"2024-08-17T17:39:14.418266+0000","last_peered":"2024-08-17T17:39:14.418266+0000","last_clean":"2024-08-17T17:39:14.418266+0000","last_became_active":"2024-08-17T17:31:36.156426+0000","last_became_peered":"2024-08-17T17:31:36.156426+0000","last_unstale":"2024-08-17T17:39:14.418266+0000","last_undegraded":"2024-08-17T17:39:14.418266+0000","last_fullsized":"2024-08-17T17:39:14.418266+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:01:48.140299+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:13.814299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":954,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421951+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:39:15.421951+0000","last_peered":"2024-08-17T17:39:15.421951+0000","last_clean":"2024-08-17T17:39:15.421951+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:39:15.421951+0000","last_undegraded":"2024-08-17T17:39:15.421951+0000","last_fullsized":"2024-08-17T17:39:15.421951+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":570,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845538+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:39:16.845538+0000","last_peered":"2024-08-17T17:39:16.845538+0000","last_clean":"2024-08-17T17:39:16.845538+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:39:16.845538+0000","last_undegraded":"2024-08-17T17:39:16.845538+0000","last_fullsized":"2024-08-17T17:39:16.845538+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":267,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422244+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:39:15.422244+0000","last_peered":"2024-08-17T17:39:15.422244+0000","last_clean":"2024-08-17T17:39:15.422244+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:39:15.422244+0000","last_undegraded":"2024-08-17T17:39:15.422244+0000","last_fullsized":"2024-08-17T17:39:15.422244+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"361'13466","reported_seq":13715,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.550914+0000","last_change":"2024-08-17T17:38:02.903957+0000","last_active":"2024-08-17T17:39:16.550914+0000","last_peered":"2024-08-17T17:39:16.550914+0000","last_clean":"2024-08-17T17:39:16.550914+0000","last_became_active":"2024-08-17T17:37:56.326921+0000","last_became_peered":"2024-08-17T17:37:56.326921+0000","last_unstale":"2024-08-17T17:39:16.550914+0000","last_undegraded":"2024-08-17T17:39:16.550914+0000","last_fullsized":"2024-08-17T17:39:16.550914+0000","mapping_epoch":274,"log_start":"280'4526","ondisk_log_start":"280'4526","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"307'11808","last_scrub_stamp":"2024-08-17T17:38:02.903808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:02.903808+0000","objects_scrubbed":1658,"log_size":8940,"log_dups_size":0,"ondisk_log_size":8940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:27.381984+0000","scrub_duration":257,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1658,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":740,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.418177+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:39:14.418177+0000","last_peered":"2024-08-17T17:39:14.418177+0000","last_clean":"2024-08-17T17:39:14.418177+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:39:14.418177+0000","last_undegraded":"2024-08-17T17:39:14.418177+0000","last_fullsized":"2024-08-17T17:39:14.418177+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":592,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845834+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:39:16.845834+0000","last_peered":"2024-08-17T17:39:16.845834+0000","last_clean":"2024-08-17T17:39:16.845834+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:39:16.845834+0000","last_undegraded":"2024-08-17T17:39:16.845834+0000","last_fullsized":"2024-08-17T17:39:16.845834+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":589,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846501+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:39:16.846501+0000","last_peered":"2024-08-17T17:39:16.846501+0000","last_clean":"2024-08-17T17:39:16.846501+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:39:16.846501+0000","last_undegraded":"2024-08-17T17:39:16.846501+0000","last_fullsized":"2024-08-17T17:39:16.846501+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"362'13629","reported_seq":13878,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846594+0000","last_change":"2024-08-17T17:38:10.314494+0000","last_active":"2024-08-17T17:39:16.846594+0000","last_peered":"2024-08-17T17:39:16.846594+0000","last_clean":"2024-08-17T17:39:16.846594+0000","last_became_active":"2024-08-17T17:37:56.313898+0000","last_became_peered":"2024-08-17T17:37:56.313898+0000","last_unstale":"2024-08-17T17:39:16.846594+0000","last_undegraded":"2024-08-17T17:39:16.846594+0000","last_fullsized":"2024-08-17T17:39:16.846594+0000","mapping_epoch":274,"log_start":"280'4368","ondisk_log_start":"280'4368","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"312'12096","last_scrub_stamp":"2024-08-17T17:38:10.314279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:10.314279+0000","objects_scrubbed":1533,"log_size":9261,"log_dups_size":0,"ondisk_log_size":9261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:16.023687+0000","scrub_duration":233,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1533,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":988,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.464169+0000","last_change":"2024-08-17T17:18:55.495463+0000","last_active":"2024-08-17T17:39:15.464169+0000","last_peered":"2024-08-17T17:39:15.464169+0000","last_clean":"2024-08-17T17:39:15.464169+0000","last_became_active":"2024-08-17T17:18:55.494902+0000","last_became_peered":"2024-08-17T17:18:55.494902+0000","last_unstale":"2024-08-17T17:39:15.464169+0000","last_undegraded":"2024-08-17T17:39:15.464169+0000","last_fullsized":"2024-08-17T17:39:15.464169+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:17.227938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":581,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.550999+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:39:16.550999+0000","last_peered":"2024-08-17T17:39:16.550999+0000","last_clean":"2024-08-17T17:39:16.550999+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:39:16.550999+0000","last_undegraded":"2024-08-17T17:39:16.550999+0000","last_fullsized":"2024-08-17T17:39:16.550999+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":628,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551797+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:39:16.551797+0000","last_peered":"2024-08-17T17:39:16.551797+0000","last_clean":"2024-08-17T17:39:16.551797+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:39:16.551797+0000","last_undegraded":"2024-08-17T17:39:16.551797+0000","last_fullsized":"2024-08-17T17:39:16.551797+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"362'13915","reported_seq":14124,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.533692+0000","last_change":"2024-08-17T17:38:41.224720+0000","last_active":"2024-08-17T17:39:15.533692+0000","last_peered":"2024-08-17T17:39:15.533692+0000","last_clean":"2024-08-17T17:39:15.533692+0000","last_became_active":"2024-08-17T17:37:24.885665+0000","last_became_peered":"2024-08-17T17:37:24.885665+0000","last_unstale":"2024-08-17T17:39:15.533692+0000","last_undegraded":"2024-08-17T17:39:15.533692+0000","last_fullsized":"2024-08-17T17:39:15.533692+0000","mapping_epoch":274,"log_start":"279'3062","ondisk_log_start":"279'3062","created":274,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"337'12985","last_scrub_stamp":"2024-08-17T17:38:41.224564+0000","last_deep_scrub":"337'12985","last_deep_scrub_stamp":"2024-08-17T17:38:41.224564+0000","last_clean_scrub_stamp":"2024-08-17T17:38:41.224564+0000","objects_scrubbed":998,"log_size":10853,"log_dups_size":0,"ondisk_log_size":10853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":7,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:29.076406+0000","scrub_duration":6.1620240859999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":998,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1077,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422200+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:39:15.422200+0000","last_peered":"2024-08-17T17:39:15.422200+0000","last_clean":"2024-08-17T17:39:15.422200+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:39:15.422200+0000","last_undegraded":"2024-08-17T17:39:15.422200+0000","last_fullsized":"2024-08-17T17:39:15.422200+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":405,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.464679+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:39:15.464679+0000","last_peered":"2024-08-17T17:39:15.464679+0000","last_clean":"2024-08-17T17:39:15.464679+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:39:15.464679+0000","last_undegraded":"2024-08-17T17:39:15.464679+0000","last_fullsized":"2024-08-17T17:39:15.464679+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":476,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551399+0000","last_change":"2024-08-17T17:22:35.156889+0000","last_active":"2024-08-17T17:39:16.551399+0000","last_peered":"2024-08-17T17:39:16.551399+0000","last_clean":"2024-08-17T17:39:16.551399+0000","last_became_active":"2024-08-17T17:22:35.153840+0000","last_became_peered":"2024-08-17T17:22:35.153840+0000","last_unstale":"2024-08-17T17:39:16.551399+0000","last_undegraded":"2024-08-17T17:39:16.551399+0000","last_fullsized":"2024-08-17T17:39:16.551399+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:07.938590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"362'13384","reported_seq":13630,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846077+0000","last_change":"2024-08-17T17:38:13.924333+0000","last_active":"2024-08-17T17:39:16.846077+0000","last_peered":"2024-08-17T17:39:16.846077+0000","last_clean":"2024-08-17T17:39:16.846077+0000","last_became_active":"2024-08-17T17:37:56.328864+0000","last_became_peered":"2024-08-17T17:37:56.328864+0000","last_unstale":"2024-08-17T17:39:16.846077+0000","last_undegraded":"2024-08-17T17:39:16.846077+0000","last_fullsized":"2024-08-17T17:39:16.846077+0000","mapping_epoch":274,"log_start":"279'3214","ondisk_log_start":"279'3214","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"316'12025","last_scrub_stamp":"2024-08-17T17:38:13.924080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:13.924080+0000","objects_scrubbed":1360,"log_size":10170,"log_dups_size":0,"ondisk_log_size":10170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:31:59.579298+0000","scrub_duration":249,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1360,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":146,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422483+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:39:15.422483+0000","last_peered":"2024-08-17T17:39:15.422483+0000","last_clean":"2024-08-17T17:39:15.422483+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:39:15.422483+0000","last_undegraded":"2024-08-17T17:39:15.422483+0000","last_fullsized":"2024-08-17T17:39:15.422483+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":874,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551466+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:39:16.551466+0000","last_peered":"2024-08-17T17:39:16.551466+0000","last_clean":"2024-08-17T17:39:16.551466+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:39:16.551466+0000","last_undegraded":"2024-08-17T17:39:16.551466+0000","last_fullsized":"2024-08-17T17:39:16.551466+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":598,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551248+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:39:16.551248+0000","last_peered":"2024-08-17T17:39:16.551248+0000","last_clean":"2024-08-17T17:39:16.551248+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:39:16.551248+0000","last_undegraded":"2024-08-17T17:39:16.551248+0000","last_fullsized":"2024-08-17T17:39:16.551248+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"362'13535","reported_seq":13782,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846780+0000","last_change":"2024-08-17T17:38:07.529499+0000","last_active":"2024-08-17T17:39:16.846780+0000","last_peered":"2024-08-17T17:39:16.846780+0000","last_clean":"2024-08-17T17:39:16.846780+0000","last_became_active":"2024-08-17T17:37:56.313596+0000","last_became_peered":"2024-08-17T17:37:56.313596+0000","last_unstale":"2024-08-17T17:39:16.846780+0000","last_undegraded":"2024-08-17T17:39:16.846780+0000","last_fullsized":"2024-08-17T17:39:16.846780+0000","mapping_epoch":274,"log_start":"280'4371","ondisk_log_start":"280'4371","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'12083","last_scrub_stamp":"2024-08-17T17:38:07.529352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:07.529352+0000","objects_scrubbed":1459,"log_size":9164,"log_dups_size":0,"ondisk_log_size":9164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:53.061774+0000","scrub_duration":454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1459,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":595,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551360+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:39:16.551360+0000","last_peered":"2024-08-17T17:39:16.551360+0000","last_clean":"2024-08-17T17:39:16.551360+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:39:16.551360+0000","last_undegraded":"2024-08-17T17:39:16.551360+0000","last_fullsized":"2024-08-17T17:39:16.551360+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":929,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.421804+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:39:14.421804+0000","last_peered":"2024-08-17T17:39:14.421804+0000","last_clean":"2024-08-17T17:39:14.421804+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:39:14.421804+0000","last_undegraded":"2024-08-17T17:39:14.421804+0000","last_fullsized":"2024-08-17T17:39:14.421804+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":592,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845995+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:39:16.845995+0000","last_peered":"2024-08-17T17:39:16.845995+0000","last_clean":"2024-08-17T17:39:16.845995+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:39:16.845995+0000","last_undegraded":"2024-08-17T17:39:16.845995+0000","last_fullsized":"2024-08-17T17:39:16.845995+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"362'13502","reported_seq":13996,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845705+0000","last_change":"2024-08-17T17:39:15.460251+0000","last_active":"2024-08-17T17:39:16.845705+0000","last_peered":"2024-08-17T17:39:16.845705+0000","last_clean":"2024-08-17T17:39:16.845705+0000","last_became_active":"2024-08-17T17:39:15.424716+0000","last_became_peered":"2024-08-17T17:39:15.424716+0000","last_unstale":"2024-08-17T17:39:16.845705+0000","last_undegraded":"2024-08-17T17:39:16.845705+0000","last_fullsized":"2024-08-17T17:39:16.845705+0000","mapping_epoch":361,"log_start":"279'3284","ondisk_log_start":"279'3284","created":274,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"315'12031","last_scrub_stamp":"2024-08-17T17:38:12.902888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:12.902888+0000","objects_scrubbed":1471,"log_size":10218,"log_dups_size":0,"ondisk_log_size":10218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:03.854575+0000","scrub_duration":188,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1471,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":585,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846848+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:39:16.846848+0000","last_peered":"2024-08-17T17:39:16.846848+0000","last_clean":"2024-08-17T17:39:16.846848+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:39:16.846848+0000","last_undegraded":"2024-08-17T17:39:16.846848+0000","last_fullsized":"2024-08-17T17:39:16.846848+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1104,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551660+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:39:16.551660+0000","last_peered":"2024-08-17T17:39:16.551660+0000","last_clean":"2024-08-17T17:39:16.551660+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:39:16.551660+0000","last_undegraded":"2024-08-17T17:39:16.551660+0000","last_fullsized":"2024-08-17T17:39:16.551660+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":583,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846100+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:39:16.846100+0000","last_peered":"2024-08-17T17:39:16.846100+0000","last_clean":"2024-08-17T17:39:16.846100+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:39:16.846100+0000","last_undegraded":"2024-08-17T17:39:16.846100+0000","last_fullsized":"2024-08-17T17:39:16.846100+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"362'13412","reported_seq":13559,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.428636+0000","last_change":"2024-08-17T17:38:27.181069+0000","last_active":"2024-08-17T17:39:16.428636+0000","last_peered":"2024-08-17T17:39:16.428636+0000","last_clean":"2024-08-17T17:39:16.428636+0000","last_became_active":"2024-08-17T17:37:56.329057+0000","last_became_peered":"2024-08-17T17:37:56.329057+0000","last_unstale":"2024-08-17T17:39:16.428636+0000","last_undegraded":"2024-08-17T17:39:16.428636+0000","last_fullsized":"2024-08-17T17:39:16.428636+0000","mapping_epoch":274,"log_start":"279'2903","ondisk_log_start":"279'2903","created":274,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"328'12195","last_scrub_stamp":"2024-08-17T17:38:27.180950+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:27.180950+0000","objects_scrubbed":1217,"log_size":10509,"log_dups_size":0,"ondisk_log_size":10509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:39.574315+0000","scrub_duration":0.152673964,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1217,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":604,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846241+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:39:16.846241+0000","last_peered":"2024-08-17T17:39:16.846241+0000","last_clean":"2024-08-17T17:39:16.846241+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:39:16.846241+0000","last_undegraded":"2024-08-17T17:39:16.846241+0000","last_fullsized":"2024-08-17T17:39:16.846241+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":335,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.464479+0000","last_change":"2024-08-17T17:39:11.405535+0000","last_active":"2024-08-17T17:39:15.464479+0000","last_peered":"2024-08-17T17:39:15.464479+0000","last_clean":"2024-08-17T17:39:15.464479+0000","last_became_active":"2024-08-17T17:39:11.397981+0000","last_became_peered":"2024-08-17T17:39:11.397981+0000","last_unstale":"2024-08-17T17:39:15.464479+0000","last_undegraded":"2024-08-17T17:39:15.464479+0000","last_fullsized":"2024-08-17T17:39:15.464479+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:08.005028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":305,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421697+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:39:15.421697+0000","last_peered":"2024-08-17T17:39:15.421697+0000","last_clean":"2024-08-17T17:39:15.421697+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:39:15.421697+0000","last_undegraded":"2024-08-17T17:39:15.421697+0000","last_fullsized":"2024-08-17T17:39:15.421697+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1f","version":"362'13625","reported_seq":17593,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845514+0000","last_change":"2024-08-17T17:38:49.694611+0000","last_active":"2024-08-17T17:39:16.845514+0000","last_peered":"2024-08-17T17:39:16.845514+0000","last_clean":"2024-08-17T17:39:16.845514+0000","last_became_active":"2024-08-17T17:38:49.645126+0000","last_became_peered":"2024-08-17T17:38:49.645126+0000","last_unstale":"2024-08-17T17:39:16.845514+0000","last_undegraded":"2024-08-17T17:39:16.845514+0000","last_fullsized":"2024-08-17T17:39:16.845514+0000","mapping_epoch":343,"log_start":"280'4371","ondisk_log_start":"280'4371","created":304,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":5,"last_scrub":"309'11981","last_scrub_stamp":"2024-08-17T17:38:04.598899+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:04.598899+0000","objects_scrubbed":1653,"log_size":9254,"log_dups_size":0,"ondisk_log_size":9254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:00.915586+0000","scrub_duration":599,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1653,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":926,"num_bytes_recovered":60620800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":268,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422408+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:39:15.422408+0000","last_peered":"2024-08-17T17:39:15.422408+0000","last_clean":"2024-08-17T17:39:15.422408+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:39:15.422408+0000","last_undegraded":"2024-08-17T17:39:15.422408+0000","last_fullsized":"2024-08-17T17:39:15.422408+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":278,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422159+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:39:15.422159+0000","last_peered":"2024-08-17T17:39:15.422159+0000","last_clean":"2024-08-17T17:39:15.422159+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:39:15.422159+0000","last_undegraded":"2024-08-17T17:39:15.422159+0000","last_fullsized":"2024-08-17T17:39:15.422159+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1e","version":"362'13430","reported_seq":16623,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846396+0000","last_change":"2024-08-17T17:38:44.357816+0000","last_active":"2024-08-17T17:39:16.846396+0000","last_peered":"2024-08-17T17:39:16.846396+0000","last_clean":"2024-08-17T17:39:16.846396+0000","last_became_active":"2024-08-17T17:38:44.320998+0000","last_became_peered":"2024-08-17T17:38:44.320998+0000","last_unstale":"2024-08-17T17:39:16.846396+0000","last_undegraded":"2024-08-17T17:39:16.846396+0000","last_fullsized":"2024-08-17T17:39:16.846396+0000","mapping_epoch":340,"log_start":"279'3214","ondisk_log_start":"279'3214","created":304,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":5,"last_scrub":"313'11944","last_scrub_stamp":"2024-08-17T17:38:10.993627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:10.993627+0000","objects_scrubbed":1486,"log_size":10216,"log_dups_size":0,"ondisk_log_size":10216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:33.563708+0000","scrub_duration":223,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1486,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":978,"num_bytes_recovered":64094208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":580,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551137+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:39:16.551137+0000","last_peered":"2024-08-17T17:39:16.551137+0000","last_clean":"2024-08-17T17:39:16.551137+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:39:16.551137+0000","last_undegraded":"2024-08-17T17:39:16.551137+0000","last_fullsized":"2024-08-17T17:39:16.551137+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:39:14.419465+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:39:14.419465+0000","last_peered":"2024-08-17T17:39:14.419465+0000","last_clean":"2024-08-17T17:39:14.419465+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:39:14.419465+0000","last_undegraded":"2024-08-17T17:39:14.419465+0000","last_fullsized":"2024-08-17T17:39:14.419465+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1d","version":"362'14327","reported_seq":18769,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845713+0000","last_change":"2024-08-17T17:38:43.793221+0000","last_active":"2024-08-17T17:39:16.845713+0000","last_peered":"2024-08-17T17:39:16.845713+0000","last_clean":"2024-08-17T17:39:16.845713+0000","last_became_active":"2024-08-17T17:38:39.327215+0000","last_became_peered":"2024-08-17T17:38:39.327215+0000","last_unstale":"2024-08-17T17:39:16.845713+0000","last_undegraded":"2024-08-17T17:39:16.845713+0000","last_fullsized":"2024-08-17T17:39:16.845713+0000","mapping_epoch":336,"log_start":"279'2954","ondisk_log_start":"279'2954","created":304,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"340'13435","last_scrub_stamp":"2024-08-17T17:38:43.793084+0000","last_deep_scrub":"340'13435","last_deep_scrub_stamp":"2024-08-17T17:38:43.793084+0000","last_clean_scrub_stamp":"2024-08-17T17:38:43.793084+0000","objects_scrubbed":917,"log_size":11373,"log_dups_size":0,"ondisk_log_size":11373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":5,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:34.530776+0000","scrub_duration":4054,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":917,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422388+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:39:15.422388+0000","last_peered":"2024-08-17T17:39:15.422388+0000","last_clean":"2024-08-17T17:39:15.422388+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:39:15.422388+0000","last_undegraded":"2024-08-17T17:39:15.422388+0000","last_fullsized":"2024-08-17T17:39:15.422388+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":547,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845211+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:39:16.845211+0000","last_peered":"2024-08-17T17:39:16.845211+0000","last_clean":"2024-08-17T17:39:16.845211+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:39:16.845211+0000","last_undegraded":"2024-08-17T17:39:16.845211+0000","last_fullsized":"2024-08-17T17:39:16.845211+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1c","version":"362'13421","reported_seq":16995,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.533573+0000","last_change":"2024-08-17T17:38:34.335318+0000","last_active":"2024-08-17T17:39:15.533573+0000","last_peered":"2024-08-17T17:39:15.533573+0000","last_clean":"2024-08-17T17:39:15.533573+0000","last_became_active":"2024-08-17T17:38:34.317469+0000","last_became_peered":"2024-08-17T17:38:34.317469+0000","last_unstale":"2024-08-17T17:39:15.533573+0000","last_undegraded":"2024-08-17T17:39:15.533573+0000","last_fullsized":"2024-08-17T17:39:15.533573+0000","mapping_epoch":333,"log_start":"279'3284","ondisk_log_start":"279'3284","created":304,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":5,"last_scrub":"322'12153","last_scrub_stamp":"2024-08-17T17:38:21.064050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:21.064050+0000","objects_scrubbed":1279,"log_size":10137,"log_dups_size":0,"ondisk_log_size":10137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:21.262735+0000","scrub_duration":452,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1279,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":73203712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":606,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846238+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:39:16.846238+0000","last_peered":"2024-08-17T17:39:16.846238+0000","last_clean":"2024-08-17T17:39:16.846238+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:39:16.846238+0000","last_undegraded":"2024-08-17T17:39:16.846238+0000","last_fullsized":"2024-08-17T17:39:16.846238+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":596,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846011+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:39:16.846011+0000","last_peered":"2024-08-17T17:39:16.846011+0000","last_clean":"2024-08-17T17:39:16.846011+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:39:16.846011+0000","last_undegraded":"2024-08-17T17:39:16.846011+0000","last_fullsized":"2024-08-17T17:39:16.846011+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.13","version":"362'12599","reported_seq":18082,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845211+0000","last_change":"2024-08-17T17:38:13.269545+0000","last_active":"2024-08-17T17:39:16.845211+0000","last_peered":"2024-08-17T17:39:16.845211+0000","last_clean":"2024-08-17T17:39:16.845211+0000","last_became_active":"2024-08-17T17:38:12.319748+0000","last_became_peered":"2024-08-17T17:38:12.319748+0000","last_unstale":"2024-08-17T17:39:16.845211+0000","last_undegraded":"2024-08-17T17:39:16.845211+0000","last_fullsized":"2024-08-17T17:39:16.845211+0000","mapping_epoch":314,"log_start":"279'2400","ondisk_log_start":"279'2400","created":289,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":5,"last_scrub":"315'11100","last_scrub_stamp":"2024-08-17T17:38:13.269289+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:13.269289+0000","objects_scrubbed":1499,"log_size":10199,"log_dups_size":0,"ondisk_log_size":10199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:00.176208+0000","scrub_duration":200,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1499,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":156,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422447+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:39:15.422447+0000","last_peered":"2024-08-17T17:39:15.422447+0000","last_clean":"2024-08-17T17:39:15.422447+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:39:15.422447+0000","last_undegraded":"2024-08-17T17:39:15.422447+0000","last_fullsized":"2024-08-17T17:39:15.422447+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":601,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846243+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:39:16.846243+0000","last_peered":"2024-08-17T17:39:16.846243+0000","last_clean":"2024-08-17T17:39:16.846243+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:39:16.846243+0000","last_undegraded":"2024-08-17T17:39:16.846243+0000","last_fullsized":"2024-08-17T17:39:16.846243+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"362'12904","reported_seq":19736,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.518024+0000","last_change":"2024-08-17T17:38:18.445589+0000","last_active":"2024-08-17T17:39:15.518024+0000","last_peered":"2024-08-17T17:39:15.518024+0000","last_clean":"2024-08-17T17:39:15.518024+0000","last_became_active":"2024-08-17T17:38:10.314402+0000","last_became_peered":"2024-08-17T17:38:10.314402+0000","last_unstale":"2024-08-17T17:39:15.518024+0000","last_undegraded":"2024-08-17T17:39:15.518024+0000","last_fullsized":"2024-08-17T17:39:15.518024+0000","mapping_epoch":312,"log_start":"279'2404","ondisk_log_start":"279'2404","created":289,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"321'11482","last_scrub_stamp":"2024-08-17T17:38:18.445320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:18.445320+0000","objects_scrubbed":1430,"log_size":10500,"log_dups_size":0,"ondisk_log_size":10500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:05.333206+0000","scrub_duration":0.314651395,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1430,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":579,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846481+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:39:16.846481+0000","last_peered":"2024-08-17T17:39:16.846481+0000","last_clean":"2024-08-17T17:39:16.846481+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:39:16.846481+0000","last_undegraded":"2024-08-17T17:39:16.846481+0000","last_fullsized":"2024-08-17T17:39:16.846481+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":582,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.550951+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:39:16.550951+0000","last_peered":"2024-08-17T17:39:16.550951+0000","last_clean":"2024-08-17T17:39:16.550951+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:39:16.550951+0000","last_undegraded":"2024-08-17T17:39:16.550951+0000","last_fullsized":"2024-08-17T17:39:16.550951+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.11","version":"362'12886","reported_seq":21187,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.520649+0000","last_change":"2024-08-17T17:38:34.126788+0000","last_active":"2024-08-17T17:39:15.520649+0000","last_peered":"2024-08-17T17:39:15.520649+0000","last_clean":"2024-08-17T17:39:15.520649+0000","last_became_active":"2024-08-17T17:37:57.315604+0000","last_became_peered":"2024-08-17T17:37:57.315604+0000","last_unstale":"2024-08-17T17:39:15.520649+0000","last_undegraded":"2024-08-17T17:39:15.520649+0000","last_fullsized":"2024-08-17T17:39:15.520649+0000","mapping_epoch":305,"log_start":"279'2600","ondisk_log_start":"279'2600","created":289,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"333'11856","last_scrub_stamp":"2024-08-17T17:38:34.126667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:34.126667+0000","objects_scrubbed":1030,"log_size":10286,"log_dups_size":0,"ondisk_log_size":10286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:08.430079+0000","scrub_duration":0.112054425,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1030,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":598,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846229+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:39:16.846229+0000","last_peered":"2024-08-17T17:39:16.846229+0000","last_clean":"2024-08-17T17:39:16.846229+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:39:16.846229+0000","last_undegraded":"2024-08-17T17:39:16.846229+0000","last_fullsized":"2024-08-17T17:39:16.846229+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":252,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421802+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:39:15.421802+0000","last_peered":"2024-08-17T17:39:15.421802+0000","last_clean":"2024-08-17T17:39:15.421802+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:39:15.421802+0000","last_undegraded":"2024-08-17T17:39:15.421802+0000","last_fullsized":"2024-08-17T17:39:15.421802+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"362'13684","reported_seq":20559,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.469027+0000","last_change":"2024-08-17T17:38:30.072764+0000","last_active":"2024-08-17T17:39:15.469027+0000","last_peered":"2024-08-17T17:39:15.469027+0000","last_clean":"2024-08-17T17:39:15.469027+0000","last_became_active":"2024-08-17T17:37:44.984903+0000","last_became_peered":"2024-08-17T17:37:44.984903+0000","last_unstale":"2024-08-17T17:39:15.469027+0000","last_undegraded":"2024-08-17T17:39:15.469027+0000","last_fullsized":"2024-08-17T17:39:15.469027+0000","mapping_epoch":300,"log_start":"280'4178","ondisk_log_start":"280'4178","created":289,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":949,"log_size":9506,"log_dups_size":0,"ondisk_log_size":9506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7633,"num_write_kb":241120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2236,"num_bytes_recovered":146472960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":104,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.551518+0000","last_change":"2024-08-17T17:39:11.399069+0000","last_active":"2024-08-17T17:39:16.551518+0000","last_peered":"2024-08-17T17:39:16.551518+0000","last_clean":"2024-08-17T17:39:16.551518+0000","last_became_active":"2024-08-17T17:39:11.398663+0000","last_became_peered":"2024-08-17T17:39:11.398663+0000","last_unstale":"2024-08-17T17:39:16.551518+0000","last_undegraded":"2024-08-17T17:39:16.551518+0000","last_fullsized":"2024-08-17T17:39:16.551518+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:28.214611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":114,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421927+0000","last_change":"2024-08-17T17:39:11.400118+0000","last_active":"2024-08-17T17:39:15.421927+0000","last_peered":"2024-08-17T17:39:15.421927+0000","last_clean":"2024-08-17T17:39:15.421927+0000","last_became_active":"2024-08-17T17:39:11.399722+0000","last_became_peered":"2024-08-17T17:39:11.399722+0000","last_unstale":"2024-08-17T17:39:15.421927+0000","last_undegraded":"2024-08-17T17:39:15.421927+0000","last_fullsized":"2024-08-17T17:39:15.421927+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:54.931057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.17","version":"362'12852","reported_seq":17292,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.507463+0000","last_change":"2024-08-17T17:38:19.339867+0000","last_active":"2024-08-17T17:39:15.507463+0000","last_peered":"2024-08-17T17:39:15.507463+0000","last_clean":"2024-08-17T17:39:15.507463+0000","last_became_active":"2024-08-17T17:38:19.320461+0000","last_became_peered":"2024-08-17T17:38:19.320461+0000","last_unstale":"2024-08-17T17:39:15.507463+0000","last_undegraded":"2024-08-17T17:39:15.507463+0000","last_fullsized":"2024-08-17T17:39:15.507463+0000","mapping_epoch":321,"log_start":"279'2600","ondisk_log_start":"279'2600","created":289,"last_epoch_clean":322,"parent":"0.0","parent_split_bits":5,"last_scrub":"290'10429","last_scrub_stamp":"2024-08-17T17:37:28.325151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:28.325151+0000","objects_scrubbed":2429,"log_size":10252,"log_dups_size":0,"ondisk_log_size":10252,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:53.739403+0000","scrub_duration":796,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2429,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1409,"num_bytes_recovered":92340224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":252,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422337+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:39:15.422337+0000","last_peered":"2024-08-17T17:39:15.422337+0000","last_clean":"2024-08-17T17:39:15.422337+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:39:15.422337+0000","last_undegraded":"2024-08-17T17:39:15.422337+0000","last_fullsized":"2024-08-17T17:39:15.422337+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":150,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422798+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:39:15.422798+0000","last_peered":"2024-08-17T17:39:15.422798+0000","last_clean":"2024-08-17T17:39:15.422798+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:39:15.422798+0000","last_undegraded":"2024-08-17T17:39:15.422798+0000","last_fullsized":"2024-08-17T17:39:15.422798+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.16","version":"362'12689","reported_seq":18697,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846507+0000","last_change":"2024-08-17T17:38:17.346526+0000","last_active":"2024-08-17T17:39:16.846507+0000","last_peered":"2024-08-17T17:39:16.846507+0000","last_clean":"2024-08-17T17:39:16.846507+0000","last_became_active":"2024-08-17T17:38:17.317265+0000","last_became_peered":"2024-08-17T17:38:17.317265+0000","last_unstale":"2024-08-17T17:39:16.846507+0000","last_undegraded":"2024-08-17T17:39:16.846507+0000","last_fullsized":"2024-08-17T17:39:16.846507+0000","mapping_epoch":319,"log_start":"279'2207","ondisk_log_start":"279'2207","created":289,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"290'10242","last_scrub_stamp":"2024-08-17T17:37:27.354341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:37:27.354341+0000","objects_scrubbed":2456,"log_size":10482,"log_dups_size":0,"ondisk_log_size":10482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:38.965951+0000","scrub_duration":764,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2456,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1436,"num_bytes_recovered":94109696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":252,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.421714+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:39:15.421714+0000","last_peered":"2024-08-17T17:39:15.421714+0000","last_clean":"2024-08-17T17:39:15.421714+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:39:15.421714+0000","last_undegraded":"2024-08-17T17:39:15.421714+0000","last_fullsized":"2024-08-17T17:39:15.421714+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":160,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.422586+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:39:15.422586+0000","last_peered":"2024-08-17T17:39:15.422586+0000","last_clean":"2024-08-17T17:39:15.422586+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:39:15.422586+0000","last_undegraded":"2024-08-17T17:39:15.422586+0000","last_fullsized":"2024-08-17T17:39:15.422586+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.15","version":"362'13065","reported_seq":16036,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.510194+0000","last_change":"2024-08-17T17:38:28.326442+0000","last_active":"2024-08-17T17:39:15.510194+0000","last_peered":"2024-08-17T17:39:15.510194+0000","last_clean":"2024-08-17T17:39:15.510194+0000","last_became_active":"2024-08-17T17:38:14.315075+0000","last_became_peered":"2024-08-17T17:38:14.315075+0000","last_unstale":"2024-08-17T17:39:15.510194+0000","last_undegraded":"2024-08-17T17:39:15.510194+0000","last_fullsized":"2024-08-17T17:39:15.510194+0000","mapping_epoch":316,"log_start":"279'2549","ondisk_log_start":"279'2549","created":289,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:34:47.430156+0000","objects_scrubbed":0,"log_size":10516,"log_dups_size":0,"ondisk_log_size":10516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7794,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1421,"num_bytes_recovered":93126656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":603,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.845538+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:39:16.845538+0000","last_peered":"2024-08-17T17:39:16.845538+0000","last_clean":"2024-08-17T17:39:16.845538+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:39:16.845538+0000","last_undegraded":"2024-08-17T17:39:16.845538+0000","last_fullsized":"2024-08-17T17:39:16.845538+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":604,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846627+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:39:16.846627+0000","last_peered":"2024-08-17T17:39:16.846627+0000","last_clean":"2024-08-17T17:39:16.846627+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:39:16.846627+0000","last_undegraded":"2024-08-17T17:39:16.846627+0000","last_fullsized":"2024-08-17T17:39:16.846627+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"362'12852","reported_seq":15906,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.522314+0000","last_change":"2024-08-17T17:38:24.436884+0000","last_active":"2024-08-17T17:39:15.522314+0000","last_peered":"2024-08-17T17:39:15.522314+0000","last_clean":"2024-08-17T17:39:15.522314+0000","last_became_active":"2024-08-17T17:38:11.313933+0000","last_became_peered":"2024-08-17T17:38:11.313933+0000","last_unstale":"2024-08-17T17:39:15.522314+0000","last_undegraded":"2024-08-17T17:39:15.522314+0000","last_fullsized":"2024-08-17T17:39:15.522314+0000","mapping_epoch":313,"log_start":"279'2362","ondisk_log_start":"279'2362","created":289,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"325'11671","last_scrub_stamp":"2024-08-17T17:38:24.436638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:47.430156+0000","last_clean_scrub_stamp":"2024-08-17T17:38:24.436638+0000","objects_scrubbed":1181,"log_size":10490,"log_dups_size":0,"ondisk_log_size":10490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:01.128859+0000","scrub_duration":0.16609647399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1181,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":268,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.427912+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:39:16.427912+0000","last_peered":"2024-08-17T17:39:16.427912+0000","last_clean":"2024-08-17T17:39:16.427912+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:39:16.427912+0000","last_undegraded":"2024-08-17T17:39:16.427912+0000","last_fullsized":"2024-08-17T17:39:16.427912+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":623,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.846228+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:39:16.846228+0000","last_peered":"2024-08-17T17:39:16.846228+0000","last_clean":"2024-08-17T17:39:16.846228+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:39:16.846228+0000","last_undegraded":"2024-08-17T17:39:16.846228+0000","last_fullsized":"2024-08-17T17:39:16.846228+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":36,"stat_sum":{"num_bytes":294912,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60294,"num_write_kb":490656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14480,"num_bytes_recovered":948830208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1376256,"data_stored":1376256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":370737,"ondisk_log_size":370737,"up":108,"acting":108,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201709,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":697808,"kb_used_data":12864,"kb_used_omap":1,"kb_used_meta":684926,"kb_avail":93047344,"statfs":{"total":95995035648,"available":95280480256,"internally_reserved":0,"allocated":13172736,"data_stored":11354844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":701364683},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922206,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313372,"kb_used_data":5064,"kb_used_omap":7,"kb_used_meta":308280,"kb_avail":93431780,"statfs":{"total":95995035648,"available":95674142720,"internally_reserved":0,"allocated":5185536,"data_stored":3375004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":315679414},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085730,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316308,"kb_used_data":4484,"kb_used_omap":7,"kb_used_meta":311800,"kb_avail":93428844,"statfs":{"total":95995035648,"available":95671136256,"internally_reserved":0,"allocated":4591616,"data_stored":2781369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":319283641},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986619,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":667280,"kb_used_data":8580,"kb_used_omap":1,"kb_used_meta":658686,"kb_avail":93077872,"statfs":{"total":95995035648,"available":95311740928,"internally_reserved":0,"allocated":8785920,"data_stored":6971623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":674494918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435511,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":735880,"kb_used_data":4476,"kb_used_omap":5,"kb_used_meta":731386,"kb_avail":93009272,"statfs":{"total":95995035648,"available":95241494528,"internally_reserved":0,"allocated":4583424,"data_stored":2780808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":748939337},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599037,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783548,"kb_used_data":7728,"kb_used_omap":7,"kb_used_meta":775800,"kb_avail":92961604,"statfs":{"total":95995035648,"available":95192682496,"internally_reserved":0,"allocated":7913472,"data_stored":6114204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":794419373},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762564,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613696,"kb_used_data":4468,"kb_used_omap":7,"kb_used_meta":609208,"kb_avail":93131456,"statfs":{"total":95995035648,"available":95366610944,"internally_reserved":0,"allocated":4575232,"data_stored":2779287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":623829409},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926090,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":744724,"kb_used_data":9160,"kb_used_omap":7,"kb_used_meta":735544,"kb_avail":93000428,"statfs":{"total":95995035648,"available":95232438272,"internally_reserved":0,"allocated":9379840,"data_stored":7566267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":753197660},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1376256,"data_stored":1376256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:22.307 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:39:22.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-08-17T17:39:22.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:22.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:22.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:22.452 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:22.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:22 smithi064 ceph-mon[104516]: pgmap v1114: 109 pgs: 109 active+clean; 4.8 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:39:22.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:22 smithi064 ceph-mon[104516]: from='client.45746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:22.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:22 smithi064 ceph-mon[109246]: pgmap v1114: 109 pgs: 109 active+clean; 4.8 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:39:22.541 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:22 smithi064 ceph-mon[109246]: from='client.45746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:22.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:22.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:22 smithi175 ceph-mon[101434]: pgmap v1114: 109 pgs: 109 active+clean; 4.8 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:39:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:22 smithi175 ceph-mon[101434]: from='client.45746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:22.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:22.701 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:22.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:22.825 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:22.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:22.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:23.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:23.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:23.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:23.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:23.298 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:23.319 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:23.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:23.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:23.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:23.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:23.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:23.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:23.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:23.895 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:23.895 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:23.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:23.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:24.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:24.062 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:24.164 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.186 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:24.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:23] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-08-17T17:39:24.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:24.307 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:24.382 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:24.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:24.524 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:24 smithi175 ceph-mon[101434]: pgmap v1115: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:24.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:24.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:24.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:24.670 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:24 smithi064 ceph-mon[104516]: pgmap v1115: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:24.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:24 smithi064 ceph-mon[109246]: pgmap v1115: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:24.679 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:24.750 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:24.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:24.803 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:24.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:24.926 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:25.049 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:25.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:25.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:25.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:25.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:25.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:25.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:25.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:25.438 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:39:25.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:25.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:25.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:25.574 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:25.575 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:39:25.575 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-08-17T17:39:25.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:39:25.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]} : dispatch 2024-08-17T17:39:25.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:25.644 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:25.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:25.655 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:25.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:25.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:25.672 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:25.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:25.688 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:25.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:25.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:25.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:25.721 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:25.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:25.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:25.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:25.753 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:25.754 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:25.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:25.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:25.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:25.919 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:25.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:26.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:26.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:26.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:26.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:26.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:26.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:26.422 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:26.437 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:39:26.437 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:39:26.437 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:39:26.437 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:39:26.437 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112111 7.d e250/248 7:b9940ed9:::benchmark_data_smithi064_210574_object13302:head v 250'7033, mlcod=0'0)", 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.13433936899997, 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.090243121999999995, 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278170+0000", 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278169+0000", 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278175+0000", 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8599999999999998e-06 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278183+0000", 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0250000000000002e-06 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278189+0000", 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2809999999999999e-06 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278223+0000", 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4010000000000001e-05 2024-08-17T17:39:26.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.278241+0000", 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8352e-05 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368371+0000", 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.090129859000000007 2024-08-17T17:39:26.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368414+0000", 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2823e-05 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112116 7.d e250/248 7:b5089113:::benchmark_data_smithi064_210574_object13307:head v 250'7034, mlcod=0'0)", 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.131566733, 2024-08-17T17:39:26.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.087496358999999996, 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.444 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280943+0000", 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280942+0000", 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.445 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280950+0000", 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1940000000000001e-06 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280959+0000", 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9550000000000004e-06 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.280965+0000", 2024-08-17T17:39:26.446 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5589999999999996e-06 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346103+0000", 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.065137729000000005 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346129+0000", 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6253000000000001e-05 2024-08-17T17:39:26.447 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368417+0000", 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.022288006999999999 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.368439+0000", 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2473e-05 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.448 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:112122 7.c e250/248 7:3111c4d1:::benchmark_data_smithi064_210574_object13313:head v 250'6950, mlcod=0'0)", 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 447.12851044400003, 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.099167975000000005, 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.449 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283999+0000", 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.283998+0000", 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.284009+0000", 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0597e-05 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346054+0000", 2024-08-17T17:39:26.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.062044729 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346060+0000", 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3489999999999997e-06 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346276+0000", 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000216386 2024-08-17T17:39:26.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.346300+0000", 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.355e-05 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.383117+0000", 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.036817132000000002 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:31:59.383167+0000", 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0305e-05 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115310 7.b e251/248 7:d3949251:::benchmark_data_smithi064_210574_object16501:head v 251'7274, mlcod=0'0)", 2024-08-17T17:39:26.454 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 442.02398429800002, 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.118751128, 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388526+0000", 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388525+0000", 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388530+0000", 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7790000000000003e-06 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388540+0000", 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0487e-05 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.457 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388547+0000", 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5429999999999999e-06 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388585+0000", 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8389000000000001e-05 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.388601+0000", 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5571999999999999e-05 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507236+0000", 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11863559999999999 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507277+0000", 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0198999999999999e-05 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115316 7.c e251/248 7:3b3fdb82:::benchmark_data_smithi064_210574_object16507:head v 251'7152, mlcod=0'0)", 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 442.02094480699998, 2024-08-17T17:39:26.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.11131099, 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.461 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391565+0000", 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.391572+0000", 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1500000000000002e-06 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493796+0000", 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.102224026 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493806+0000", 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9459999999999993e-06 2024-08-17T17:39:26.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493889+0000", 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3380000000000005e-05 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493916+0000", 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6599000000000001e-05 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.502831+0000", 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.008915002 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.502876+0000", 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5386999999999998e-05 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.35217.0:115322 7.b e251/248 7:dc3829f1:::benchmark_data_smithi064_210574_object16513:head v 251'7275, mlcod=0'0)", 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 442.01659335800002, 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.113719247, 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395916+0000", 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.395923+0000", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2509999999999997e-06 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493591+0000", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.097667976000000004 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.493608+0000", 2024-08-17T17:39:26.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7342000000000001e-05 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.497919+0000", 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043104470000000002 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.497943+0000", 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4584999999999999e-05 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.507284+0000", 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0093404249999999994 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:32:04.509636+0000", 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023520199999999998 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:45490 8.5 8:acb8b83f:::benchmark_data_smithi064_242541_object45489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:23.558320+0000", 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 242.854189502, 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.082735383999999995, 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 45490 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.471 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558320+0000", 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558320+0000", 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558316+0000", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999957 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558358+0000", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1835000000000002e-05 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558370+0000", 2024-08-17T17:39:26.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2428e-05 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558378+0000", 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558429+0000", 2024-08-17T17:39:26.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1276999999999997e-05 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558578+0000", 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014874300000000001 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.558731+0000", 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000153697 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.560976+0000", 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022445059999999998 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.561376+0000", 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00040051399999999998 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.640957+0000", 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.079580810000000002 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.641013+0000", 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5671000000000003e-05 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:23.641056+0000", 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2920999999999998e-05 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:57313 8.5 8:a2f12edb:::benchmark_data_smithi064_242541_object57312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:31.057670+0000", 2024-08-17T17:39:26.478 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 235.35483985400001, 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.081615518999999997, 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 57313 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.479 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057670+0000", 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057670+0000", 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057666+0000", 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999967 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057714+0000", 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7176999999999999e-05 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057724+0000", 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0321e-05 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057729+0000", 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7570000000000004e-06 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057771+0000", 2024-08-17T17:39:26.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2070999999999998e-05 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057886+0000", 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011527399999999999 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.057982+0000", 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6248000000000004e-05 2024-08-17T17:39:26.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.059579+0000", 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001596735 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.060102+0000", 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00052297499999999996 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.484 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.139182+0000", 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.079080046000000001 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.139240+0000", 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7788000000000003e-05 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:31.139286+0000", 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5689999999999998e-05 2024-08-17T17:39:26.485 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85799 8.5 8:a98281b0:::benchmark_data_smithi064_242541_object85798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.857003+0000", 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 211.55550666100001, 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.17486732499999999, 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.486 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85799 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857003+0000", 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857003+0000", 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857002+0000", 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857053+0000", 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1076999999999999e-05 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857065+0000", 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2177999999999999e-05 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857071+0000", 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1650000000000003e-06 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857108+0000", 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6239000000000003e-05 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857207+0000", 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9143999999999998e-05 2024-08-17T17:39:26.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.857308+0000", 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101753 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.859803+0000", 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002494554 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.860285+0000", 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00048213200000000002 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.031770+0000", 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.17148498000000001 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.031816+0000", 2024-08-17T17:39:26.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6356999999999997e-05 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.031871+0000", 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4030999999999998e-05 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:85821 8.5 8:a78f10b2:::benchmark_data_smithi064_242541_object85820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-17T17:39:26.493 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:54.895142+0000", 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 211.517367646, 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.13694588899999999, 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 85821 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.494 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895142+0000", 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895142+0000", 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895140+0000", 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895198+0000", 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7222999999999997e-05 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895209+0000", 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1816e-05 2024-08-17T17:39:26.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895214+0000", 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.459e-06 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895251+0000", 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7626000000000002e-05 2024-08-17T17:39:26.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895408+0000", 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015629700000000001 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.895566+0000", 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015823700000000001 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.897792+0000", 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022260919999999998 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:54.898092+0000", 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00030043600000000001 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.032014+0000", 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.133921385 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.032056+0000", 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2508e-05 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:55.032088+0000", 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1720999999999997e-05 2024-08-17T17:39:26.500 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90933 8.5 8:a4de45c9:::benchmark_data_smithi064_242541_object90932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.806264+0000", 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.60624546099999, 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.278967878, 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.501 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90933 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806264+0000", 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806264+0000", 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806263+0000", 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806461+0000", 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000198257 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806472+0000", 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0560000000000001e-05 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806478+0000", 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1310000000000003e-06 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806507+0000", 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9187999999999999e-05 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806593+0000", 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6378999999999994e-05 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806695+0000", 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101635 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084529+0000", 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27783408300000001 2024-08-17T17:39:26.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084575+0000", 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6227999999999999e-05 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085151+0000", 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00057583299999999999 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085194+0000", 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2412e-05 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085232+0000", 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8649999999999998e-05 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.508 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:90935 8.5 8:a406ed59:::benchmark_data_smithi064_242541_object90934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.806866+0000", 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.605643837, 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27845950800000002, 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 90935 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.509 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806866+0000", 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806866+0000", 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806865+0000", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806897+0000", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1798000000000003e-05 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806908+0000", 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1202e-05 2024-08-17T17:39:26.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.806914+0000", 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4880000000000004e-06 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.809508+0000", 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025932989999999999 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.809629+0000", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121607 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.809761+0000", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013143399999999999 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084559+0000", 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27479781800000003 2024-08-17T17:39:26.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084582+0000", 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3623000000000001e-05 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085263+0000", 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068091699999999996 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085295+0000", 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2008999999999998e-05 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.085326+0000", 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0343999999999999e-05 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:90940 8.7 e282/274 8:eecee401:::benchmark_data_smithi064_242541_object90939:head v 282'5712, mlcod=282'5710)", 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.811534+0000", 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.60097585, 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27333506099999999, 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.516 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811534+0000", 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811534+0000", 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811533+0000", 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811568+0000", 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4402e-05 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811579+0000", 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1541e-05 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811586+0000", 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7139999999999996e-06 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811642+0000", 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5937000000000003e-05 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811662+0000", 2024-08-17T17:39:26.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0267000000000001e-05 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084837+0000", 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.273174747 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084869+0000", 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2279e-05 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.520 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:90932 8.6 e282/274 8:6973c697:::benchmark_data_smithi064_242541_object90931:head v 282'5630, mlcod=282'5627)", 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.811908+0000", 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.60060146000001, 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.279271832, 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811908+0000", 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811908+0000", 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811908+0000", 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811946+0000", 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7898000000000002e-05 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811956+0000", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9839999999999996e-06 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.811962+0000", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0090000000000004e-06 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.812025+0000", 2024-08-17T17:39:26.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2966000000000007e-05 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.812042+0000", 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.772e-05 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.090751+0000", 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.278708277 2024-08-17T17:39:26.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.091180+0000", 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00042962499999999999 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop_reply(client.45488.0:90933 8.5 e282/274 ondisk, result = 0)", 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.813912+0000", 2024-08-17T17:39:26.526 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.59859833799999, 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27062060500000001, 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.813912+0000", 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.813912+0000", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.813910+0000", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.813922+0000", 2024-08-17T17:39:26.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1583000000000001e-05 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084476+0000", 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27055380299999998 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084484+0000", 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4689999999999997e-06 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084512+0000", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7617999999999999e-05 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084522+0000", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8349999999999999e-06 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084532+0000", 2024-08-17T17:39:26.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0360000000000001e-05 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:90939 8.6 e282/274 8:6663ab88:::benchmark_data_smithi064_242541_object90938:head v 282'5631, mlcod=282'5627)", 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.814135+0000", 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.598374785, 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28254367600000002, 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.531 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.814135+0000", 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.814135+0000", 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.814134+0000", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.814199+0000", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4623999999999997e-05 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084597+0000", 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27039793000000001 2024-08-17T17:39:26.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084605+0000", 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084639+0000", 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4059e-05 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084663+0000", 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4023e-05 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.096632+0000", 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011969527000000001 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.096679+0000", 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6399000000000001e-05 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:90948 8.3 e282/274 8:ce2fc90b:::benchmark_data_smithi064_242541_object90947:head v 282'5530, mlcod=282'5528)", 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:35:59.819293+0000", 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 206.593216466, 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.26863911200000001, 2024-08-17T17:39:26.536 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.819293+0000", 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.819293+0000", 2024-08-17T17:39:26.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.819281+0000", 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999881 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: pgmap v1116: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:39:26.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/544357122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: pgmap v1116: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:39:26.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:26 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/544357122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:35:59.819328+0000", 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6962999999999997e-05 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084699+0000", 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.26537050499999998 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084709+0000", 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0052e-05 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084754+0000", 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.515e-05 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.084776+0000", 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2149000000000001e-05 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.087846+0000", 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030700910000000001 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:36:00.087932+0000", 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6244999999999998e-05 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164890 8.5 8:a88a0acf:::benchmark_data_smithi064_242541_object42397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 122.562463803, 2024-08-17T17:39:26.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45790435600000001, 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164890 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850044+0000", 2024-08-17T17:39:26.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850052+0000", 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7350000000000002e-06 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850071+0000", 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8816000000000001e-05 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850080+0000", 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6379999999999993e-06 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850138+0000", 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8700999999999999e-05 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850281+0000", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142476 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850400+0000", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000118967 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.857056+0000", 2024-08-17T17:39:26.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0066560730000000002 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.861294+0000", 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042383389999999998 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.307933+0000", 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.44663844800000002 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.307950+0000", 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7654000000000001e-05 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164890 8.15 8:a88a0acf:::benchmark_data_smithi064_242541_object42397:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 122.373462687, 2024-08-17T17:39:26.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.87629330800000005, 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164890 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:39:26.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039042+0000", 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999943 2024-08-17T17:39:26.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039056+0000", 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4637999999999999e-05 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039085+0000", 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8266999999999999e-05 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039095+0000", 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0818999999999999e-05 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.308685+0000", 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.26958931800000002 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:39:26.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.308694+0000", 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8880000000000003e-06 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.915250+0000", 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.60655672699999996 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.915340+0000", 2024-08-17T17:39:26.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0210000000000005e-05 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:197932 8.7 e310/289 8:e068bd0e:::benchmark_data_smithi064_242541_object75439:head v 310'11359, mlcod=310'11357)", 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 78.661151320000002, 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25881922899999998, 2024-08-17T17:39:26.558 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:39:26.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751358+0000", 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751367+0000", 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0070000000000003e-06 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.007946+0000", 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25657962299999998 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.007961+0000", 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4599e-05 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:39:26.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.008006+0000", 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4879999999999997e-05 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.008025+0000", 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8760999999999999e-05 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.010153+0000", 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021286819999999998 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.010178+0000", 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4434999999999999e-05 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:39:26.563 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:39:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.565 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:26.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:26.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:26.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:26.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:26.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: pgmap v1116: 109 pgs: 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/403026927' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 2]}]': finished 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:39:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:26 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/544357122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:26.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:26.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:27.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:27.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:27.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:39:27.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:27.313 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:27.414 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:27.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:27.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:27.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:27.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:27.684 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:27.786 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:27.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:27.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:27 smithi064 ceph-mon[104516]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:39:27.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:27 smithi064 ceph-mon[109246]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:39:27.909 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:27.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:27 smithi175 ceph-mon[101434]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:39:27.930 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:28.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:28.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:28.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:28.156 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:28.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:28.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:28.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:28.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:28.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:28.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:28.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:28.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:28 smithi064 ceph-mon[104516]: pgmap v1119: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:28.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:28 smithi064 ceph-mon[109246]: pgmap v1119: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:28.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:28.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:28.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:28.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:28.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:28.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:28 smithi175 ceph-mon[101434]: pgmap v1119: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:28.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:29.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.049 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:29.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:29.111 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:29.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:29.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:39:29.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:29.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:29.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:29.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:29.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:29.462 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pg_num to 11 2024-08-17T17:39:29.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:29.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:29.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:29.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:29.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:29.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2904000969' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:29.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:29.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2904000969' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:29.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:29.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:29.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:30.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2904000969' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:30.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:39:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:30.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:30.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:30.291 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:30.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:30.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:30.471 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-08-17T17:39:30.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-08-17T17:39:30.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:30.541 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:30.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:30.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:30.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:30.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:30.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:30.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:30.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[104516]: pgmap v1120: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:30.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:39:30.886 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[104516]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:39:30.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[109246]: pgmap v1120: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:30.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:39:30.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:30 smithi064 ceph-mon[109246]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:39:30.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:30.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:30 smithi175 ceph-mon[101434]: pgmap v1120: 109 pgs: 1 activating, 1 peering, 107 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:30.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:39:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:30 smithi175 ceph-mon[101434]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:39:30.917 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:30.998 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:31.005 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:31.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:31.022 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:31.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:31.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:31.175 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:31.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:31.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:31.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:31.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:31.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:31.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:31.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:31.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:31.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:31 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:31.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:31 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:31.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:31.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:31.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:31.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:31 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:31.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.058 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:32.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:32.159 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:32.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:32.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:32.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:32.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:32.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:32.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:32.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[104516]: pgmap v1122: 109 pgs: 1 activating, 108 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:32.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:39:32.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[104516]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:39:32.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[109246]: pgmap v1122: 109 pgs: 1 activating, 108 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:32.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:39:32.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:32 smithi064 ceph-mon[109246]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:39:32.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:32.687 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:32.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:32.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:32 smithi175 ceph-mon[101434]: pgmap v1122: 109 pgs: 1 activating, 108 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:32 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:39:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:32 smithi175 ceph-mon[101434]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:39:32.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:32.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:33.044 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:33.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:33.197 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:33.298 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.323 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:33.425 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:33.449 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:33.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:33.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:33.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:33.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:33 smithi064 ceph-mon[109246]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:39:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:33 smithi064 ceph-mon[104516]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:39:33.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:33.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:33.832 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:33.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:33 smithi175 ceph-mon[101434]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:39:33.934 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.958 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:34.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:34.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:34.186 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:33] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-08-17T17:39:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:34.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:34.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:34.460 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:34.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:34.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:34.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:34 smithi064 ceph-mon[104516]: pgmap v1125: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:34 smithi064 ceph-mon[109246]: pgmap v1125: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:34.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:34 smithi175 ceph-mon[101434]: pgmap v1125: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:34.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:34.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:35.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.086 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:35.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:35.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:35.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:35.472 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:39:35.472 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:39:35.472 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-17T17:39:35.473 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:39:35.473 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:39:35.473 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-17T17:39:35.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:35.587 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:35.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:35.646 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:35.663 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:35.663 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:35.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:35.679 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:35.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:35.691 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:35.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:35.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:35.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:35.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:35.712 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:35.712 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:35.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:35.729 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:35.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:35.745 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:35.762 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:35.762 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:35.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.867 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:35.867 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:35.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:35.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:36.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:36.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:36.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:36.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:36.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:36.453 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:36.492 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' removed 2024-08-17T17:39:36.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:36.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:36.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:36.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:36.826 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:36 smithi175 ceph-mon[101434]: pgmap v1126: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:36.826 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/841123397' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.826 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:36 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.826 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/141370431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:36.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:36.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[104516]: pgmap v1126: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/841123397' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/141370431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[109246]: pgmap v1126: 119 pgs: 10 unknown, 109 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/841123397' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:39:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/141370431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:36.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:36.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:36.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:37.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.113 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:37.141 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:39:37.141 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:39:37.142 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-08-17T17:39:37.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-08-17T17:39:37.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:37.245 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:37.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:37.363 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:39:37.363 INFO:teuthology.orchestra.run.smithi175.stdout: "success": "" 2024-08-17T17:39:37.363 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:39:37.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:37.471 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:37.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:37.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:37.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:37 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:39:37.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:37 smithi175 ceph-mon[101434]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:39:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:39:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[104516]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:39:37.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:39:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[109246]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:39:37.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:39:37.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:37.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:37.862 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:37.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.987 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:38.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:39:38.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:38.253 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:38.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:38.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:38.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:38.501 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:38.600 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:38.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:38.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[104516]: pgmap v1128: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:39:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[104516]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:39:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[104516]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:39:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[109246]: pgmap v1128: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:39:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[109246]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:39:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:38 smithi064 ceph-mon[109246]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:39:38.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:38.871 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:38 smithi175 ceph-mon[101434]: pgmap v1128: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:39:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:38 smithi175 ceph-mon[101434]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:39:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:38 smithi175 ceph-mon[101434]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:39:38.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.995 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:38.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:39.096 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:39.117 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:39.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:39.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:39.339 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:39:39.341 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:39.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:39.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.487 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:39.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:39.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:39.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:39.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:39.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:39.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: 1.0 scrub starts 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: 1.0 scrub ok 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2684093109' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:39.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:39 smithi175 ceph-mon[101434]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:39:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: 1.0 scrub starts 2024-08-17T17:39:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: 1.0 scrub ok 2024-08-17T17:39:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2684093109' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[109246]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: 1.0 scrub starts 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: 1.0 scrub ok 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2684093109' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:39 smithi064 ceph-mon[104516]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:39:39.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:39.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:40.083 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:40.104 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:40.206 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:40.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:40.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:40.354 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:40.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:40.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:40.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:40.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[109246]: pgmap v1131: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[109246]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:39:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[104516]: pgmap v1131: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:40 smithi064 ceph-mon[104516]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:39:40.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.725 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:40.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:40 smithi175 ceph-mon[101434]: pgmap v1131: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:40 smithi175 ceph-mon[101434]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:39:40.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:40.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:41.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:41.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:41.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:41.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:41.350 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:41.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.475 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:41.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:41.488 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:41.488 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:41.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:41.598 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:41.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:41.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:41 smithi175 ceph-mon[101434]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:39:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:41.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:41 smithi064 ceph-mon[104516]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:39:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:41.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:41 smithi064 ceph-mon[109246]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:39:41.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:41.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:41.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:41.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:41.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:41.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:42.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:42.093 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:42.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:42.195 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:39:42.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:42.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:42.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:42.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:42.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:42.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:42.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[109246]: pgmap v1134: 83 pgs: 4 peering, 1 active+remapped, 4 active+undersized+remapped, 74 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:39:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:39:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:39:42.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[104516]: pgmap v1134: 83 pgs: 4 peering, 1 active+remapped, 4 active+undersized+remapped, 74 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:39:42.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:39:42.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:42 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:39:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:42.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:42.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.839 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:42.857 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:42 smithi175 ceph-mon[101434]: pgmap v1134: 83 pgs: 4 peering, 1 active+remapped, 4 active+undersized+remapped, 74 active+clean; 4.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T17:39:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:42 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:39:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:42 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:39:42.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:43.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:39:43.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:43.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:43.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:43.205 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:39:43.206 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:39:43.206 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:39:43.206 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.288328 2024-08-17T17:39:43.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 0.288327591220483 2024-08-17T17:39:43.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:43.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:43.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:43.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:43.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:43.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:43 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:39:43.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:43 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:39:43.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:43 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:39:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:43 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:39:43.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:43.774 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:43 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:39:43.774 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:43 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:39:43.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:43.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:43.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:44.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:44.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:44.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:44.147 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:44.228 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:43] "GET /metrics HTTP/1.1" 200 34342 "" "Prometheus/2.51.0" 2024-08-17T17:39:44.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:44.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:44.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:44.493 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:44.497 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:44.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:44.539 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:44.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:44.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:44 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:39:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:44 smithi175 ceph-mon[101434]: pgmap v1136: 83 pgs: 5 peering, 1 active+remapped, 3 active+undersized+remapped, 74 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-17T17:39:44.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:44.758 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:44 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:39:44.758 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:44 smithi064 ceph-mon[104516]: pgmap v1136: 83 pgs: 5 peering, 1 active+remapped, 3 active+undersized+remapped, 74 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-17T17:39:44.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:44 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:39:44.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:44 smithi064 ceph-mon[109246]: pgmap v1136: 83 pgs: 5 peering, 1 active+remapped, 3 active+undersized+remapped, 74 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-17T17:39:44.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:44.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:44.891 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:44.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:45.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:45.135 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:45.135 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:45.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:45.151 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:45.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:45.168 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:45.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:45.185 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:45.185 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:45.201 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:45.201 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:45.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:45.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:45.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:45.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:45.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:45.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:45.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:45.510 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:45.526 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' created 2024-08-17T17:39:45.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:45.634 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:45.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:45.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:45.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:39:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2874599388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:45.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2874599388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:45.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2874599388' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:45.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:46.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:46.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:46.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.136 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:46.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:46.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:46.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:46.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:46.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:46.468 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:46.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:46.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:46.616 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:46.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:46.744 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:46.769 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:46.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[104516]: pgmap v1137: 83 pgs: 5 peering, 2 active+undersized+remapped, 76 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:39:46.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:39:46.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[104516]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:39:46.852 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[109246]: pgmap v1137: 83 pgs: 5 peering, 2 active+undersized+remapped, 76 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:39:46.852 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:39:46.852 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:46 smithi064 ceph-mon[109246]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:39:46.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:46.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:46.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:46 smithi175 ceph-mon[101434]: pgmap v1137: 83 pgs: 5 peering, 2 active+undersized+remapped, 76 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T17:39:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1559262240' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:39:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:46 smithi175 ceph-mon[101434]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:39:46.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:47.018 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:47.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:47.244 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:47.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:47.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:47.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:47.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:47.519 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:47.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:47.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:47.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:47.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:47.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:47.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:47 smithi175 ceph-mon[101434]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:39:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:47 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:39:47.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:47 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:39:47.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[104516]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:39:47.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:39:47.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:39:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[109246]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:39:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:39:47.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:47 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:39:47.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:48.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:48.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:48.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:48.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:48.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:48.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:48.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:48.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:48.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:48.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:48.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:48.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:48.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:48.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:48.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:48.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:48.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:48.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:39:48.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[104516]: pgmap v1140: 99 pgs: 7 unknown, 9 creating+peering, 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:48.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:39:48.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[104516]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:39:48.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:39:48.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:39:48.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[109246]: pgmap v1140: 99 pgs: 7 unknown, 9 creating+peering, 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:48.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:39:48.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[109246]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:39:48.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:48 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:39:48.881 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:48.908 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:48 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:39:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:48 smithi175 ceph-mon[101434]: pgmap v1140: 99 pgs: 7 unknown, 9 creating+peering, 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:48 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:39:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:48 smithi175 ceph-mon[101434]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:39:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:48 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:39:49.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:49.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:49.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.159 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:49.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:49.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:49.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:49.390 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:49.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:49.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-08-17T17:39:49.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:49.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:49.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:49.680 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:49.739 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:49 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:39:49.739 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:49 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:39:49.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:49.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:49.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:49.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:49 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:39:49.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.034 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:50.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:50.187 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:50.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:50.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:50.313 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:50.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:50.457 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:50.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:50.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:50.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:50.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:50 smithi064 ceph-mon[104516]: pgmap v1142: 99 pgs: 3 unknown, 9 creating+peering, 2 active+clean+scrubbing, 85 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:50.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:50 smithi064 ceph-mon[109246]: pgmap v1142: 99 pgs: 3 unknown, 9 creating+peering, 2 active+clean+scrubbing, 85 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:50.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:50.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:50.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:50 smithi175 ceph-mon[101434]: pgmap v1142: 99 pgs: 3 unknown, 9 creating+peering, 2 active+clean+scrubbing, 85 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:50.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:50.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:50.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:51.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.082 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:51.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:51.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:51.250 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:51.251 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:51.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:51.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:51.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:51.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:51.561 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:39:51.561 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 0.288328 (8188952) 2024-08-17T17:39:51.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:51.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:51.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:51.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:51.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4270213348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4084299171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4270213348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4084299171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.818 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:51.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:51 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4270213348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:51 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4084299171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483} : dispatch 2024-08-17T17:39:51.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:51.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:52.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:52.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:52.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:52.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:52.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:52.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:52.329 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:52.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:52.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:52.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:52.583 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:52.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:52.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: pgmap v1143: 99 pgs: 3 creating+peering, 96 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483}]': finished 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: pgmap v1143: 99 pgs: 3 creating+peering, 96 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483}]': finished 2024-08-17T17:39:52.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:39:52.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:52 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:39:52.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:52.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:52.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:52.854 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:52.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: pgmap v1143: 99 pgs: 3 creating+peering, 96 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.288327591220483}]': finished 2024-08-17T17:39:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:39:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:52 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:39:52.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:52.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:53.080 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.107 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:53.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:53.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:53.231 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:53.282 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96083 7.7 7:ed271dc4:::benchmark_data_smithi064_210574_object96082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.13130463099998, 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.254662196, 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:39:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96083 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145031+0000", 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145032+0000", 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9999999999999997e-07 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145053+0000", 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1299000000000001e-05 2024-08-17T17:39:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145061+0000", 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9370000000000004e-06 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145067+0000", 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0650000000000004e-06 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145108+0000", 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0983999999999999e-05 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145208+0000", 2024-08-17T17:39:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100515 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3", 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.145294+0000", 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5458999999999996e-05 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.157015+0000", 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011720932999999999 2024-08-17T17:39:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.175024+0000", 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018009237000000001 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.175037+0000", 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3448999999999999e-05 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156505+0000", 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98146811099999998 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156522+0000", 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6762e-05 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.399566+0000", 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243043882 2024-08-17T17:39:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.399693+0000", 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012686300000000001 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96119 7.7 7:e600f8d0:::benchmark_data_smithi064_210574_object96118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.07669552300001, 2024-08-17T17:39:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2232309840000002, 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96119 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:39:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199640+0000", 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199641+0000", 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3399999999999997e-07 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199661+0000", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0596000000000001e-05 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199669+0000", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.199675+0000", 2024-08-17T17:39:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6300000000000003e-06 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202654+0000", 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029795360000000001 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202665+0000", 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0156999999999999e-05 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156615+0000", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95395067300000003 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156624+0000", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8119999999999997e-06 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.402759+0000", 2024-08-17T17:39:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2461353340000001 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.402932+0000", 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017221700000000001 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.403048+0000", 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116597 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.408939+0000", 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058903360000000004 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409660+0000", 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072114200000000003 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.418271+0000", 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086108550000000006 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.418335+0000", 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4623999999999997e-05 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422871+0000", 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045356609999999999 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35217.0:96126 7.7 7:ef424558:::benchmark_data_smithi064_210574_object96125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.07334064399998, 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2244136810000001, 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.300 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35217", 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 96126 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202995+0000", 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.202996+0000", 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0599999999999999e-07 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203017+0000", 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1101999999999999e-05 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203028+0000", 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1287999999999999e-05 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203035+0000", 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203087+0000", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2159e-05 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:35.203096+0000", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6260000000000001e-06 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156669+0000", 2024-08-17T17:39:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95357339399999996 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:36.156676+0000", 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3450000000000002e-06 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.405915+0000", 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2492391650000001 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.406033+0000", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118504 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.406143+0000", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109416 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409095+0000", 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029522350000000001 2024-08-17T17:39:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.409794+0000", 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069938699999999999 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422928+0000", 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013134043999999999 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.422985+0000", 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6724000000000001e-05 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:31:37.427408+0000", 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044234900000000004 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198079 8.16 8:6b1f037e:::benchmark_data_smithi064_242541_object75586:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316507+0000", 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.95982861500001, 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.144481657, 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198079 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316507+0000", 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316507+0000", 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316507+0000", 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.29e-07 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316508+0000", 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4169999999999999e-06 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316512+0000", 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2729999999999998e-06 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316514+0000", 2024-08-17T17:39:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5770000000000001e-06 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316563+0000", 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8729000000000001e-05 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316566+0000", 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3909999999999998e-06 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.455253+0000", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1386861559999999 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.455380+0000", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012753800000000001 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.455745+0000", 2024-08-17T17:39:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000365235 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460486+0000", 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047408859999999997 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460606+0000", 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119264 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460660+0000", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4778000000000003e-05 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460677+0000", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7022000000000001e-05 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460717+0000", 2024-08-17T17:39:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9541e-05 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460765+0000", 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8316999999999998e-05 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460830+0000", 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4821999999999997e-05 2024-08-17T17:39:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.460988+0000", 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015848199999999999 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198108 8.16 8:6e1184ae:::benchmark_data_smithi064_242541_object75615:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316526+0000", 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.959809205, 2024-08-17T17:39:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1504017559999999, 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198108 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316526+0000", 2024-08-17T17:39:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316526+0000", 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316526+0000", 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.28e-07 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316528+0000", 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3480000000000001e-06 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316531+0000", 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6820000000000001e-06 2024-08-17T17:39:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316536+0000", 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3379999999999998e-06 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316580+0000", 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3844000000000002e-05 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316583+0000", 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3050000000000001e-06 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.458000+0000", 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1414169279999999 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.458103+0000", 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001031 2024-08-17T17:39:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.458413+0000", 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030988499999999998 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461055+0000", 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026424370000000001 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461111+0000", 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5272000000000002e-05 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461126+0000", 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4966999999999999e-05 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461176+0000", 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0881999999999998e-05 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461223+0000", 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6913000000000003e-05 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.466665+0000", 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054420390000000001 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.466739+0000", 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3669000000000002e-05 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.466928+0000", 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018891900000000001 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198137 8.16 8:695d0c1c:::benchmark_data_smithi064_242541_object75644:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316545+0000", 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.959790185, 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1565152000000001, 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198137 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316545+0000", 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316545+0000", 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316546+0000", 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.53e-07 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316547+0000", 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2270000000000001e-06 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316550+0000", 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.517e-06 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316554+0000", 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0999999999999997e-06 2024-08-17T17:39:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316612+0000", 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7652999999999998e-05 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316618+0000", 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6969999999999998e-06 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461273+0000", 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1446555680000001 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461346+0000", 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2800999999999996e-05 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.461717+0000", 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037066099999999999 2024-08-17T17:39:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.466983+0000", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052659480000000003 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467027+0000", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3996999999999999e-05 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467073+0000", 2024-08-17T17:39:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6530999999999999e-05 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467114+0000", 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0908000000000002e-05 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467157+0000", 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2565000000000003e-05 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.472860+0000", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057037279999999999 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.472925+0000", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4719000000000002e-05 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473060+0000", 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135327 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198151 8.16 8:6ba4615e:::benchmark_data_smithi064_242541_object75658:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316559+0000", 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.95977616, 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.160091618, 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198151 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316559+0000", 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316559+0000", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316559+0000", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.23e-07 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316561+0000", 2024-08-17T17:39:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0219999999999999e-06 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316564+0000", 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1870000000000001e-06 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316566+0000", 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5979999999999999e-06 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316640+0000", 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3833999999999998e-05 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316644+0000", 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5360000000000001e-06 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.463922+0000", 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1472780419999999 2024-08-17T17:39:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.464044+0000", 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122559 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.464376+0000", 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000331477 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467198+0000", 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028219819999999998 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467241+0000", 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2908999999999999e-05 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467280+0000", 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9539999999999998e-05 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467321+0000", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0781999999999997e-05 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473228+0000", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059071430000000001 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476380+0000", 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003151671 2024-08-17T17:39:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476445+0000", 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5300000000000002e-05 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476651+0000", 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000205813 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198172 8.16 8:694faf12:::benchmark_data_smithi064_242541_object75679:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316578+0000", 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.95975710499999, 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1603313609999999, 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198172 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316578+0000", 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316578+0000", 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316579+0000", 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.35e-07 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316580+0000", 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.331e-06 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316584+0000", 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6389999999999998e-06 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316586+0000", 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9100000000000001e-06 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316665+0000", 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8980999999999995e-05 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316669+0000", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9179999999999996e-06 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467350+0000", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.150680967 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467459+0000", 2024-08-17T17:39:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108226 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.467869+0000", 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041002399999999999 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473121+0000", 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052520570000000001 2024-08-17T17:39:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473182+0000", 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1161000000000006e-05 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473269+0000", 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7335000000000003e-05 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473349+0000", 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0173000000000006e-05 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476664+0000", 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033143399999999998 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476711+0000", 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7698999999999999e-05 2024-08-17T17:39:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476769+0000", 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7269999999999999e-05 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476910+0000", 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014109499999999999 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:198178 8.16 8:6bf2b0ae:::benchmark_data_smithi064_242541_object75685:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:10.316607+0000", 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.95972809, 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.171738135, 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 198178 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316607+0000", 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316607+0000", 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316608+0000", 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0199999999999998e-07 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316609+0000", 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.689e-06 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316614+0000", 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3200000000000001e-06 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316617+0000", 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1219999999999999e-06 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316683+0000", 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6608999999999999e-05 2024-08-17T17:39:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:10.316687+0000", 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4709999999999999e-06 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.470233+0000", 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15354662 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.470368+0000", 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013413800000000001 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5,6,7", 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.470774+0000", 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040685299999999999 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473311+0000", 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025366159999999998 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.473391+0000", 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0130999999999996e-05 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476361+0000", 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029696480000000001 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476670+0000", 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000308774 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.476967+0000", 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029772700000000002 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.488138+0000", 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01117055 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.488200+0000", 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1950000000000001e-05 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:11.488345+0000", 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014561500000000001 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:220255 8.1f 8:ff12f45a:::benchmark_data_smithi064_242541_object97762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e343)", 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:48.333240+0000", 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.943095688, 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.342070882, 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 220255 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333240+0000", 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333240+0000", 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333241+0000", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9699999999999994e-07 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333246+0000", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.763e-06 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333257+0000", 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.169e-05 2024-08-17T17:39:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333264+0000", 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.922e-06 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333319+0000", 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4993e-05 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.333326+0000", 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9750000000000001e-06 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.345480+0000", 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012154148 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.345490+0000", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9159999999999996e-06 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.670472+0000", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3249817290000001 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.670581+0000", 2024-08-17T17:39:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010885 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675257+0000", 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046762269999999998 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675311+0000", 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3671999999999997e-05 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:220277 8.1f 8:face96b3:::benchmark_data_smithi064_242541_object97784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:48.349791+0000", 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.926544706000001, 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3257714439999999, 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 220277 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349791+0000", 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349791+0000", 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349792+0000", 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.638e-06 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349798+0000", 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4569999999999996e-06 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349815+0000", 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7201999999999999e-05 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349824+0000", 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6270000000000006e-06 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349881+0000", 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7005000000000001e-05 2024-08-17T17:39:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.349896+0000", 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5068999999999999e-05 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.670789+0000", 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3208935230000001 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.670871+0000", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1762999999999996e-05 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.671181+0000", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030977000000000001 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.674934+0000", 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037535950000000002 2024-08-17T17:39:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675028+0000", 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3925000000000006e-05 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675444+0000", 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041578200000000001 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675477+0000", 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2750000000000003e-05 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675562+0000", 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5338000000000002e-05 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:220316 8.1f 8:fe7d8ade:::benchmark_data_smithi064_242541_object97823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:48.383494+0000", 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.892841786000005, 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2948223649999999, 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 220316 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383494+0000", 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383494+0000", 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383495+0000", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5900000000000005e-07 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383499+0000", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1699999999999999e-06 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383510+0000", 2024-08-17T17:39:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1119e-05 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383517+0000", 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8290000000000002e-06 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383571+0000", 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4772999999999999e-05 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.383579+0000", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7270000000000007e-06 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.672582+0000", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2890031239999999 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.672658+0000", 2024-08-17T17:39:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5181999999999999e-05 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.672909+0000", 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025115700000000001 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.674981+0000", 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020725850000000001 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675357+0000", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037582099999999998 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.678206+0000", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028493949999999998 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.678245+0000", 2024-08-17T17:39:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8815e-05 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.678316+0000", 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0709000000000004e-05 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:220371 8.1f 8:fb68b24f:::benchmark_data_smithi064_242541_object97878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:48.521388+0000", 2024-08-17T17:39:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.754946941, 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159560119, 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 220371 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521388+0000", 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521388+0000", 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521390+0000", 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.796e-06 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521393+0000", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9249999999999999e-06 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521403+0000", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7690000000000008e-06 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521409+0000", 2024-08-17T17:39:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8270000000000003e-06 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521516+0000", 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010720100000000001 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:48.521528+0000", 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2211e-05 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.673789+0000", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1522610959999999 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.673857+0000", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7452000000000004e-05 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.674044+0000", 2024-08-17T17:39:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018730900000000001 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675365+0000", 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013207799999999999 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.675612+0000", 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024717700000000001 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.680848+0000", 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052362520000000003 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.680882+0000", 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3309000000000002e-05 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.680949+0000", 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7014999999999994e-05 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221001 8.21 8:8407315b:::benchmark_data_smithi064_242541_object98508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.544192649999999, 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.682936939, 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221001 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4499999999999997e-07 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732147+0000", 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.269e-06 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732156+0000", 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0149999999999992e-06 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732160+0000", 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3449999999999999e-06 2024-08-17T17:39:53.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732206+0000", 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6066000000000002e-05 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732213+0000", 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9280000000000004e-06 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351505+0000", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61929169299999998 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351518+0000", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2826000000000001e-05 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409384+0000", 2024-08-17T17:39:53.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057866347 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409504+0000", 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119895 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409764+0000", 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026007200000000003 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.411743+0000", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019789 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414636+0000", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002893161 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414860+0000", 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000223816 2024-08-17T17:39:53.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414939+0000", 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8861000000000003e-05 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414991+0000", 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2759000000000001e-05 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415080+0000", 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8341e-05 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221048 8.21 8:87156b27:::benchmark_data_smithi064_242541_object98555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.766037+0000", 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.510298038999998, 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6495826419999999, 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.404 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221048 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766037+0000", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766037+0000", 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766039+0000", 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0279999999999999e-06 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766052+0000", 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2542e-05 2024-08-17T17:39:53.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766071+0000", 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9091000000000001e-05 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.766079+0000", 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4109999999999996e-06 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.767448+0000", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001368586 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.767459+0000", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1416999999999999e-05 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351545+0000", 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.58408519800000003 2024-08-17T17:39:53.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351553+0000", 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9019999999999999e-06 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.411795+0000", 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.060242328 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.411920+0000", 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012538800000000001 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.412116+0000", 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019565399999999999 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414870+0000", 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027539209999999999 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415127+0000", 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000257264 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415322+0000", 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019459100000000001 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415442+0000", 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120239 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415504+0000", 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2178e-05 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415620+0000", 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011590399999999999 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221212 8.21 8:8719d222:::benchmark_data_smithi064_242541_object98719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.906476+0000", 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.369859642999998, 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5093318070000001, 2024-08-17T17:39:53.413 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221212 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906476+0000", 2024-08-17T17:39:53.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906476+0000", 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906477+0000", 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8999999999999995e-07 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906481+0000", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2309999999999998e-06 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906494+0000", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3030000000000001e-05 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906501+0000", 2024-08-17T17:39:53.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3540000000000002e-06 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906569+0000", 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7237999999999996e-05 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.906581+0000", 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2381000000000001e-05 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351578+0000", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44499697100000002 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351585+0000", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5989999999999996e-06 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.413199+0000", 2024-08-17T17:39:53.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0616134159999999 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.413331+0000", 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013221199999999999 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.413563+0000", 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023234500000000001 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415262+0000", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001698867 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415385+0000", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122471 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415685+0000", 2024-08-17T17:39:53.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000300494 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415724+0000", 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8618999999999997e-05 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415754+0000", 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0099000000000001e-05 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415808+0000", 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.359e-05 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221246 8.21 8:8636d5c3:::benchmark_data_smithi064_242541_object98753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.934162+0000", 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.342173312, 2024-08-17T17:39:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4861479040000001, 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221246 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934162+0000", 2024-08-17T17:39:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934162+0000", 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934164+0000", 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.339e-06 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934171+0000", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3199999999999996e-06 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934192+0000", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1016e-05 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934201+0000", 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5920000000000007e-06 2024-08-17T17:39:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934277+0000", 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5809999999999994e-05 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.934289+0000", 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2189e-05 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351610+0000", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.41732017799999999 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351618+0000", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.089e-06 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415845+0000", 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0642270229999999 2024-08-17T17:39:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415959+0000", 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113823 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.416169+0000", 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000210922 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.419995+0000", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038254790000000001 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.420060+0000", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5522e-05 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.420088+0000", 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7866999999999999e-05 2024-08-17T17:39:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.420138+0000", 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0130000000000003e-05 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.420195+0000", 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6509999999999999e-05 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.420310+0000", 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011509499999999999 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221252 8.21 8:848bf06d:::benchmark_data_smithi064_242541_object98759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.942207+0000", 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.334128173000003, 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4908043550000001, 2024-08-17T17:39:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221252 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942207+0000", 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942207+0000", 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942209+0000", 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4100000000000001e-06 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942214+0000", 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8770000000000001e-06 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942226+0000", 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2679e-05 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942233+0000", 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5910000000000001e-06 2024-08-17T17:39:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942293+0000", 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0303999999999997e-05 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.942303+0000", 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.978e-06 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351643+0000", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.40934022799999997 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351659+0000", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5430999999999998e-05 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.418099+0000", 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06644052 2024-08-17T17:39:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.418219+0000", 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120048 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.418481+0000", 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026184100000000002 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.428401+0000", 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099197569999999995 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.428454+0000", 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3470999999999997e-05 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.428500+0000", 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5299000000000001e-05 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.432836+0000", 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043367559999999998 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.432897+0000", 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1017000000000002e-05 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433012+0000", 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011414800000000001 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221265 8.21 8:878b1326:::benchmark_data_smithi064_242541_object98772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.955091+0000", 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.321244905, 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4850680249999999, 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221265 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955091+0000", 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955091+0000", 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955091+0000", 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5799999999999999e-07 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955094+0000", 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9909999999999998e-06 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955102+0000", 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.038e-06 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955106+0000", 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3089999999999997e-06 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955166+0000", 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9329000000000001e-05 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.955177+0000", 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0808000000000001e-05 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351688+0000", 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.39651152099999998 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351696+0000", 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0609999999999996e-06 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.426066+0000", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0743700249999999 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.426217+0000", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015103999999999999 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.426645+0000", 2024-08-17T17:39:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042743000000000002 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433022+0000", 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063776680000000004 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433078+0000", 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5624999999999997e-05 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433131+0000", 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3199999999999999e-05 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.439916+0000", 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067853130000000003 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.439991+0000", 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4376999999999996e-05 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.440159+0000", 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167732 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221271 8.21 8:84707e09:::benchmark_data_smithi064_242541_object98778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.962946+0000", 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.313389078999997, 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.47761396, 2024-08-17T17:39:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221271 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962946+0000", 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962946+0000", 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962947+0000", 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9199999999999999e-07 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:39:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962952+0000", 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4599999999999996e-06 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962966+0000", 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4351e-05 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.962972+0000", 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2169999999999999e-06 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.963026+0000", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4042000000000002e-05 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.963034+0000", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0520000000000002e-06 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351726+0000", 2024-08-17T17:39:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.388691856 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351734+0000", 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0160000000000008e-06 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.428684+0000", 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.07694998 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.428802+0000", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117186 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.429094+0000", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029274599999999998 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433197+0000", 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041028289999999997 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.433211+0000", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3512999999999999e-05 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.437336+0000", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041255889999999998 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.440377+0000", 2024-08-17T17:39:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030412939999999999 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.440442+0000", 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4072000000000006e-05 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.440560+0000", 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011876500000000001 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:39:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:39:53.459 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:39:53.480 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:53.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:53.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:53.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:53.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:53.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:53.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:53.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:53 smithi175 ceph-mon[101434]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:39:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:53 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:39:53.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:53 smithi064 ceph-mon[104516]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:39:53.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:53 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:39:53.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:53 smithi064 ceph-mon[109246]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:39:53.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:53 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:39:53.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:53.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:53.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:54.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:54.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:54.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:54.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:54.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:54.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:54.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:54.424 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:39:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:39:53] "GET /metrics HTTP/1.1" 200 34342 "" "Prometheus/2.51.0" 2024-08-17T17:39:54.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:54.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:54.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:54.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:54.615 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:54.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:39:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[104516]: pgmap v1146: 99 pgs: 99 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:39:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:39:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[109246]: pgmap v1146: 99 pgs: 99 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:54 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:39:54.717 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:54.743 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:54.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:54.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:54.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:54 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:39:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:54 smithi175 ceph-mon[101434]: pgmap v1146: 99 pgs: 99 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:54 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:39:54.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:54.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:55.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.125 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:55.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:55.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:55.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:55.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:55.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:55.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:55.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:55.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.683 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:39:55.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:55.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:55.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:39:55.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:55.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:55.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:39:55.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:55.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:55.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:55.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:55.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:55 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:39:55.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:39:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:56.004 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:56.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:56.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:56.154 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:56.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:56.278 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:56.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:56.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:56.503 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:56.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:56.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:56.655 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:56.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:56.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:56.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:56.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:56 smithi064 ceph-mon[104516]: pgmap v1147: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:56.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:56 smithi064 ceph-mon[109246]: pgmap v1147: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:56.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:56 smithi175 ceph-mon[101434]: pgmap v1147: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:57.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:57.032 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:57.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:57.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:57.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:57.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:57.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:57.408 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:57.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:57.532 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:57.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:57.656 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:57.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:57.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:57.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:57 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:39:57.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:57 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:39:57.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:57.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:57.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:57 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:39:58.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:58.027 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:58.086 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:39:58.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:39:58.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:58.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:58.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:58.273 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:58.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:58.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:58.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:58.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:58 smithi064 ceph-mon[104516]: pgmap v1148: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:58 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:39:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:58 smithi064 ceph-mon[109246]: pgmap v1148: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:58.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:58 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:39:58.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:58.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:58.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:39:58.810 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:58.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:39:58.817 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:58.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:39:58.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:58.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:39:58.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:58.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:39:58.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:58.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:58.888 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:39:58.888 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:58.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:58.905 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:39:58.905 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:58.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:39:58.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:58 smithi175 ceph-mon[101434]: pgmap v1148: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:39:58.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:58 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:39:58.921 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:39:58.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:58.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.018 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:59.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:59.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:59.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:59.265 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:59.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:59.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:59.390 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:59.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:59.593 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:59.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:59.637 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:59.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2230732789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:39:59 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:39:59.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2230732789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:39:59 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:39:59.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:39:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.843 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:59.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:59.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:39:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:59.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2230732789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:39:59 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:39:59.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:00.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:00.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:00.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:00.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:00.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:00.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.394 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:00.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:00.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:00.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:00.589 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-08-17T17:40:00.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:00.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:00.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:00.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:00.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:00.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:40:00.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: pgmap v1149: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:00.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: application not enabled on pool 'unique_pool_4' 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/618934561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: pgmap v1149: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:00.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: application not enabled on pool 'unique_pool_4' 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/618934561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:00.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:40:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: pgmap v1149: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:40:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T17:40:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:40:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: application not enabled on pool 'unique_pool_4' 2024-08-17T17:40:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:40:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/618934561' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:40:00.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:01.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:01.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:01.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:01.251 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:01.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:01.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:01.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:01.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:01.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:01.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:01.655 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:01.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:01.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:01.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:01.904 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:01.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:01 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:40:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:01 smithi175 ceph-mon[101434]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:40:01.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:01 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:40:01.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:01 smithi064 ceph-mon[104516]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:40:01.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:01 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:40:01.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:01 smithi064 ceph-mon[109246]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:40:02.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:02.031 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:02.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:02.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:02.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:02.282 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:02.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:02.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:02.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:02.532 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:02.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:02.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:02.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:02.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:02.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[104516]: pgmap v1151: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:02.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[109246]: pgmap v1151: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:02.905 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:02 smithi175 ceph-mon[101434]: pgmap v1151: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:02 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:40:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:02 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:40:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:02.946 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-08-17T17:40:02.949 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:40:02.949 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:40:03.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:40:03.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:40:03.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_275711 2024-08-17T17:40:03.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:40:03.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:40:03.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:03.051 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:03.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:03.183 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:03.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:03.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:03.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:03.452 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:03.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:03.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:03.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:03.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:03.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:03.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:03 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:40:03.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:03 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:40:03.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:03 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:40:03.972 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:04.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3927 3911 244.422 244.438 0.00202375 0.00407955 2024-08-17T17:40:04.004 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:04.106 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.144 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.145 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:04.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:04.281 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:03] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-08-17T17:40:04.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:04.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:04.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:04.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:04.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:04.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:04.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:04.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:04.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:04 smithi175 ceph-mon[101434]: pgmap v1152: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:04 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:40:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:04.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:04.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[104516]: pgmap v1152: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:04.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:40:04.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:04.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:04.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:04.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[109246]: pgmap v1152: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 4.5 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:40:04.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:40:04.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:04.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:04.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:04.981 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:05.001 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 7059 7043 220.041 195.75 0.00454259 0.00453493 2024-08-17T17:40:05.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:05.118 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:05.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:05.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:05.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:05.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:05.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:05.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:05.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:05.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:05.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:05.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:05.800 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:05.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:05 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:40:05.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:05 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:40:05.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:05.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:05 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:40:05.942 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:06.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 10248 10232 213.126 199.312 0.00426559 0.00468303 2024-08-17T17:40:06.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:06.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:06.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:06.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:06.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:06.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:06.356 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:06.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:06.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:06.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:06.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:06.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:06.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:06 smithi064 ceph-mon[104516]: pgmap v1153: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 69 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s wr, 103 op/s 2024-08-17T17:40:06.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:06 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:40:06.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:06 smithi064 ceph-mon[109246]: pgmap v1153: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 69 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s wr, 103 op/s 2024-08-17T17:40:06.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:06 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:40:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:06 smithi175 ceph-mon[101434]: pgmap v1153: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 69 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s wr, 103 op/s 2024-08-17T17:40:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:06 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:40:06.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:06.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:07.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 13008 12992 202.966 172.5 0.00413121 0.00492001 2024-08-17T17:40:07.014 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:07.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:07.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:07.133 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:07.133 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:07.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.186 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:07.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:07.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.320 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:07.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:07.421 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:07.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:07.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:07.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:07.645 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:40:07.693 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.724 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:07.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:07.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:07.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:07 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:40:07.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:40:07.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:07 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:40:07.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:40:07.858 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:07 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:40:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:40:07.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:07.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:08.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15864 15848 198.07 178.5 0.00656628 0.0050441 2024-08-17T17:40:08.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:08.139 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:08.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:08.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:08.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:08.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:08.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:08.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:40:08.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:08.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:08.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[104516]: pgmap v1154: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 521 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:40:08.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:40:08.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[104516]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:40:08.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[109246]: pgmap v1154: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 521 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:40:08.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:40:08.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:08 smithi064 ceph-mon[109246]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:40:08.666 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:08.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:08.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:08.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:08.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:08 smithi175 ceph-mon[101434]: pgmap v1154: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 521 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:40:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2100547708' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:40:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:08 smithi175 ceph-mon[101434]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:40:08.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:08.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:09.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 18637 18621 193.931 173.312 0.00883431 0.005151 2024-08-17T17:40:09.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:09.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:40:09.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.132 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:09.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:09.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:09.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:09.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:09.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:09.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.661 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:09.800 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:09.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:09 smithi064 ceph-mon[104516]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:40:09.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:09 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:40:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:09 smithi064 ceph-mon[109246]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:40:09.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:09 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:40:09.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 20883 20867 186.279 140.375 0.00627127 0.00536242 2024-08-17T17:40:10.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:10.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:10.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:09 smithi175 ceph-mon[101434]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:40:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:09 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:40:10.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.320 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:10.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:10.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:10.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:10.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:10.617 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:10.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:10.750 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:10.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:10.888 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[104516]: pgmap v1157: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 747 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:40:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:40:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[109246]: pgmap v1157: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 747 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:10.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:11.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 22989 22973 179.446 131.625 0.00368395 0.00552533 2024-08-17T17:40:11.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:11.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:11.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:11.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:10 smithi175 ceph-mon[101434]: pgmap v1157: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 747 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-17T17:40:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:10 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:40:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:10 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:40:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:11.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:11.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:11.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:11.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:11.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:11.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:11.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:11.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:11.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:11.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:11.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:11.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:11 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:40:11.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:11 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:40:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:11.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:12.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 25019 25003 173.604 126.875 0.00531344 0.00575468 2024-08-17T17:40:12.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:12.104 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:12.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:11 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:40:12.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:12.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:12.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:12.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:12.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:12.510 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:12.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:12.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:12.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:12.772 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:12.854 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:12.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:12.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:12.916 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:12.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:12 smithi064 ceph-mon[104516]: pgmap v1158: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:40:12.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:12 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:40:12.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:12 smithi064 ceph-mon[109246]: pgmap v1158: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:40:12.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:12 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:40:13.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 27128 27112 169.423 131.812 0.0047629 0.00589585 2024-08-17T17:40:13.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:13.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:13.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:12 smithi175 ceph-mon[101434]: pgmap v1158: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:40:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:12 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:40:13.182 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:13.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:13.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:13.313 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:13.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:13.451 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:13.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:13.580 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:13.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:13.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:13.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:13.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:13.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:13.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:13.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:14.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 29037 29021 164.866 119.312 0.00282079 0.00606001 2024-08-17T17:40:14.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:14.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:14.245 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:14.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:14.384 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:13] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.51.0" 2024-08-17T17:40:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.387 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:14.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:14.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:14.519 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:14.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:14.658 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:14.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:14.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:14.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:14.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:14.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:15.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 30616 30600 159.351 98.6875 0.00321432 0.00627044 2024-08-17T17:40:15.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:15.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:15.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:14 smithi175 ceph-mon[101434]: pgmap v1159: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-17T17:40:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:14 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:40:15.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:15.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:14 smithi064 ceph-mon[104516]: pgmap v1159: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-17T17:40:15.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:14 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:40:15.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:14 smithi064 ceph-mon[109246]: pgmap v1159: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-17T17:40:15.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:14 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:40:15.219 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:15.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:15.363 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:15.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:15.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:15.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:15.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:15.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:15.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:15.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:15.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:16.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 32781 32765 157.498 135.312 0.00629338 0.00634386 2024-08-17T17:40:16.033 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:16.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:16.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:15 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:40:16.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:15 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:40:16.175 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:15 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:40:16.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:15 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:40:16.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:15 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:40:16.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:15 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:40:16.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.312 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:16.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:16.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:16.451 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:16.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:16.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:16.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:16.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:16.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:16.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:16.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:16 smithi064 ceph-mon[104516]: pgmap v1160: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-08-17T17:40:16.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:16 smithi064 ceph-mon[109246]: pgmap v1160: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-08-17T17:40:16.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:17.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:17.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 34977 34961 156.051 137.25 0.00470336 0.00640272 2024-08-17T17:40:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:17.105 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:17.134 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:17.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:16 smithi175 ceph-mon[101434]: pgmap v1160: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-08-17T17:40:17.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:17.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:17.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:17.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:17.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:17.552 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:17.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:17.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:17.696 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:17.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:17.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:17 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:40:17.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:17 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:40:17.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:17 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:40:17.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:17 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:40:17.829 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:17.931 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:17.959 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 36981 36965 153.877 125.25 0.00213259 0.00649069 2024-08-17T17:40:18.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:18.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:17 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:40:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:17 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:40:18.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:18.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:18.326 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:18.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:18.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:18.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:18.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:18.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:40:18.521 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:18.566 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:40:18.567 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:18.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:40:18.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:18.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:18.647 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:40:18.647 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:18.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:18.683 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:40:18.683 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:18.720 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:40:18.720 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:18.741 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:40:18.750 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.753 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:40:18.754 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:18.781 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.800 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:40:18.800 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:18.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:18.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[104516]: pgmap v1161: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1344886181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/956759246' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[109246]: pgmap v1161: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1344886181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/956759246' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:18 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:19.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 38788 38772 151.32 112.938 0.00546726 0.00660355 2024-08-17T17:40:19.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.064 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:18 smithi175 ceph-mon[101434]: pgmap v1161: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:40:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1344886181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/956759246' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:18 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:19.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:19.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:19.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:19.488 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:19.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:19.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:19.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:19.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:19.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:19.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:19.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:19.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:20.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:20.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 40325 40309 148.072 96.0625 0.00931627 0.00674766 2024-08-17T17:40:20.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:20.117 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:20.149 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:20.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:19 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:20.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:19 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:19 smithi175 ceph-mon[101434]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:40:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:19 smithi175 ceph-mon[101434]: 1.1 scrub starts 2024-08-17T17:40:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:19 smithi175 ceph-mon[101434]: 1.1 scrub ok 2024-08-17T17:40:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[104516]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:40:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[104516]: 1.1 scrub starts 2024-08-17T17:40:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[104516]: 1.1 scrub ok 2024-08-17T17:40:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[109246]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:40:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[109246]: 1.1 scrub starts 2024-08-17T17:40:20.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:19 smithi064 ceph-mon[109246]: 1.1 scrub ok 2024-08-17T17:40:20.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:20.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:20.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:20.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:20.532 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:20.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:20.665 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:20.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:20.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:20.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.842 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:20.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:20.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:20.974 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:40:20.974 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:40:20.974 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:40:20.974 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:40:20.974 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:40:20.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: pgmap v1163: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:40:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.9 scrub starts 2024-08-17T17:40:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.9 scrub ok 2024-08-17T17:40:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.6 scrub starts 2024-08-17T17:40:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.6 scrub ok 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.5 scrub starts 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[104516]: 1.5 scrub ok 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: pgmap v1163: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.9 scrub starts 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.9 scrub ok 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.6 scrub starts 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.6 scrub ok 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.5 scrub starts 2024-08-17T17:40:20.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:20 smithi064 ceph-mon[109246]: 1.5 scrub ok 2024-08-17T17:40:20.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:20.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:21.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 15 42258 42243 146.561 120.875 0.00800085 0.00681797 2024-08-17T17:40:21.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:21.108 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:21.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: pgmap v1163: 99 pgs: 3 active+clean+scrubbing, 96 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.9 scrub starts 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.9 scrub ok 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.6 scrub starts 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.6 scrub ok 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.5 scrub starts 2024-08-17T17:40:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:20 smithi175 ceph-mon[101434]: 1.5 scrub ok 2024-08-17T17:40:21.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:21.274 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:21.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:21.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.413 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:21.552 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:21.653 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:21.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:21.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:40:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[104516]: 1.4 scrub ok 2024-08-17T17:40:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[104516]: 1.2 scrub starts 2024-08-17T17:40:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[104516]: 1.2 scrub ok 2024-08-17T17:40:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:40:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[109246]: 1.4 scrub ok 2024-08-17T17:40:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[109246]: 1.2 scrub starts 2024-08-17T17:40:21.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:21 smithi064 ceph-mon[109246]: 1.2 scrub ok 2024-08-17T17:40:21.956 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 44216 44200 145.285 122.312 0.00751514 0.00687802 2024-08-17T17:40:22.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:22.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:21 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:40:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:21 smithi175 ceph-mon[101434]: 1.4 scrub ok 2024-08-17T17:40:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:21 smithi175 ceph-mon[101434]: 1.2 scrub starts 2024-08-17T17:40:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:21 smithi175 ceph-mon[101434]: 1.2 scrub ok 2024-08-17T17:40:22.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:22.218 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:22.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:22.363 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:22.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:22.496 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:22.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:22.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:22.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:22.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.896 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:22.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:22.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: pgmap v1164: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:40:22.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:40:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.7 scrub ok 2024-08-17T17:40:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:40:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.3 scrub ok 2024-08-17T17:40:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.a scrub starts 2024-08-17T17:40:22.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[104516]: 1.a scrub ok 2024-08-17T17:40:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: pgmap v1164: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:40:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:40:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.7 scrub ok 2024-08-17T17:40:22.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:40:22.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.3 scrub ok 2024-08-17T17:40:22.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.a scrub starts 2024-08-17T17:40:22.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:22 smithi064 ceph-mon[109246]: 1.a scrub ok 2024-08-17T17:40:22.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:23.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:40:23.015944+0000 min lat: 0.0012219 max lat: 0.230089 avg lat: 0.00696281 2024-08-17T17:40:23.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:40:23.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 45972 45956 143.509 109.75 0.0116105 0.00696281 2024-08-17T17:40:23.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:23.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: pgmap v1164: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.7 scrub ok 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.3 scrub ok 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.a scrub starts 2024-08-17T17:40:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:22 smithi175 ceph-mon[101434]: 1.a scrub ok 2024-08-17T17:40:23.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:23.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:23.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:23.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:23.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:23.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:23.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:23.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:23.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:23.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:23.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:24.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 47706 47690 141.836 108.375 0.00782359 0.00704523 2024-08-17T17:40:24.087 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:24.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:24.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:24.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:23] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.51.0" 2024-08-17T17:40:24.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:24.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:24.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:24.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:24.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:24.543 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:24.646 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:24.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:24.683 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:24.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:24.839 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:24 smithi064 ceph-mon[104516]: pgmap v1165: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-08-17T17:40:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:24 smithi064 ceph-mon[109246]: pgmap v1165: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-08-17T17:40:24.942 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:24.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:25.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 49464 49448 140.379 109.875 0.0156555 0.00711759 2024-08-17T17:40:25.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:25.099 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:25.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:24 smithi175 ceph-mon[101434]: pgmap v1165: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-08-17T17:40:25.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:25.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:25.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:25.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:25.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:25.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:25.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:25.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:25.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:25.785 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:25.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:25.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:26.016 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 51138 51122 138.825 104.625 0.0120895 0.0071983 2024-08-17T17:40:26.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:26.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:26.061 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:26.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:26.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:26.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:26.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:26.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:40:26.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:26.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:26.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:26.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:26.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:26.649 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:26.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:26.785 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:26.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:40:26.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:26.917 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:27.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 52744 52728 137.223 100.375 0.00775709 0.00728221 2024-08-17T17:40:27.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:27.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:26 smithi175 ceph-mon[101434]: pgmap v1166: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-08-17T17:40:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:26 smithi175 ceph-mon[101434]: 1.8 scrub starts 2024-08-17T17:40:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:26 smithi175 ceph-mon[101434]: 1.8 scrub ok 2024-08-17T17:40:27.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:27.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[104516]: pgmap v1166: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-08-17T17:40:27.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[104516]: 1.8 scrub starts 2024-08-17T17:40:27.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[104516]: 1.8 scrub ok 2024-08-17T17:40:27.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[109246]: pgmap v1166: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-08-17T17:40:27.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[109246]: 1.8 scrub starts 2024-08-17T17:40:27.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:26 smithi064 ceph-mon[109246]: 1.8 scrub ok 2024-08-17T17:40:27.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:27.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:27.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:27.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:27.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:27.457 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:27.477 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:27.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:27.567 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:27.570 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:27.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.765 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:27.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:27.866 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:27.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:27.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:28.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 54155 54139 135.262 88.1875 0.013866 0.00738748 2024-08-17T17:40:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:28.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:28.135 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:28.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:28.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:28.306 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:28.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:28.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:28.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:28.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:28.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:28.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:28.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:28.818 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:40:28.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:28.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:29.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 15 55602 55587 133.541 90.5 0.00650337 0.00748344 2024-08-17T17:40:29.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.107 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:29.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:28 smithi175 ceph-mon[101434]: pgmap v1167: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-08-17T17:40:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1842388032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[104516]: pgmap v1167: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-08-17T17:40:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1842388032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[109246]: pgmap v1167: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-08-17T17:40:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1842388032' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:40:29.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.387 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.489 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:29.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:29.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:29.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:29.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:29.803 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:29.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:29.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:40:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:29 smithi064 ceph-mon[104516]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:40:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:40:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:29 smithi064 ceph-mon[109246]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:40:30.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 57248 57232 132.403 102.812 0.00953547 0.00754772 2024-08-17T17:40:30.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:30.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:40:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:29 smithi175 ceph-mon[101434]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:40:30.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:30.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:30.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:30.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:30.447 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:30.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:30.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:30.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:30.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:30.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:30.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:30.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:30.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:30.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 58764 58748 131.059 94.75 0.0138202 0.00762472 2024-08-17T17:40:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:31.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:30 smithi175 ceph-mon[101434]: pgmap v1169: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.78k op/s 2024-08-17T17:40:31.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:30 smithi064 ceph-mon[104516]: pgmap v1169: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.78k op/s 2024-08-17T17:40:31.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:30 smithi064 ceph-mon[109246]: pgmap v1169: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.78k op/s 2024-08-17T17:40:31.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:31.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:31.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:31.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:31.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:31.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:31.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:31.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:31.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:40:31.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:31.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:31.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:31.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:32.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:32.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 60363 60347 129.985 99.9375 0.0107317 0.00768836 2024-08-17T17:40:32.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:32.140 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:32.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:32.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:32.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:32.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:32.573 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:32.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:32.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:32.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:32.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:32.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:32 smithi064 ceph-mon[104516]: pgmap v1170: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s 2024-08-17T17:40:32.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:32 smithi064 ceph-mon[109246]: pgmap v1170: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s 2024-08-17T17:40:32.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:32.988 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:33.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 61855 61839 128.76 93.25 0.0476276 0.00775345 2024-08-17T17:40:33.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:33.126 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:32 smithi175 ceph-mon[101434]: pgmap v1170: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s 2024-08-17T17:40:33.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:33.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:33.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:33.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.507 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.536 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:33.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:33.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.684 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.952 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:33.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:34.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 63300 63284 127.52 90.3125 0.0102761 0.00783665 2024-08-17T17:40:34.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.187 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:33] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-08-17T17:40:34.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:34.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:34.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.725 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.863 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:35.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:35.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 64760 64744 126.387 91.25 0.0156921 0.00790753 2024-08-17T17:40:35.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:34 smithi064 ceph-mon[104516]: pgmap v1171: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2024-08-17T17:40:35.084 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:34 smithi064 ceph-mon[109246]: pgmap v1171: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2024-08-17T17:40:35.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:34 smithi175 ceph-mon[101434]: pgmap v1171: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2024-08-17T17:40:35.177 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.279 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:35.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:35.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:35.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:35.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:35.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:35.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:35.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:35.703 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:35.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:35.845 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:35.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:35.860 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:40:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:35.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:35.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:35.979 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:35.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:36.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 66019 66003 124.942 78.6875 0.00705111 0.00799829 2024-08-17T17:40:36.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.122 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:36.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:36.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:36.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:36.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:36.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:36.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:36.687 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:36.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.828 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.930 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:36.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:37.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 67448 67432 123.894 89.3125 0.0152764 0.00806553 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[104516]: pgmap v1172: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[104516]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[109246]: pgmap v1172: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:37.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:36 smithi064 ceph-mon[109246]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:40:37.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:36 smithi175 ceph-mon[101434]: pgmap v1172: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-08-17T17:40:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:36 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:36 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:40:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3278959366' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:36 smithi175 ceph-mon[101434]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:40:37.209 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:37.238 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:37.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.369 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.422 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:40:37.422 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:37.465 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:40:37.466 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:37.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:37.503 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:40:37.504 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:37.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:40:37.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:37.569 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:40:37.569 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:37.620 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:40:37.620 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:37.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:40:37.669 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:37.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:40:37.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:37.787 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.817 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:37.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:38.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 68739 68723 122.66 80.6875 0.00312733 0.00813897 2024-08-17T17:40:38.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:38.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:38.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/422745624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/422745624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/422745624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:38.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:38.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:38.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.371 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:38.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:38.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:38.537 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:38.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:38.666 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:38.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:38.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:38.958 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:38.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:39.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 70111 70095 121.635 85.75 0.00881485 0.00821617 2024-08-17T17:40:39.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:38 smithi175 ceph-mon[101434]: pgmap v1174: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-17T17:40:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:38 smithi064 ceph-mon[104516]: pgmap v1174: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-17T17:40:39.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:38 smithi064 ceph-mon[109246]: pgmap v1174: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-17T17:40:39.190 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:39.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.507 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:39.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:39.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:39.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:39.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:39.784 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:39.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:40:40.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 71559 71543 120.793 90.5 0.0147169 0.00827308 2024-08-17T17:40:40.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:40.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:40.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:40.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:39 smithi175 ceph-mon[101434]: pgmap v1175: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:40:40.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:39 smithi064 ceph-mon[109246]: pgmap v1175: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:40:40.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:39 smithi064 ceph-mon[104516]: pgmap v1175: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:40:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:40.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:40:40.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:40.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:40.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:40.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:40.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:40.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:40.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:40.723 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:40.825 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:40.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:41.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 72775 72759 119.615 76 0.00449206 0.00835517 2024-08-17T17:40:41.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:41.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:41.223 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:41.254 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:41.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:41.389 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:41.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:41.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:41 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:41.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:41 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:41.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:41.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:41 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:41.491 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:41.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:41.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:41.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.653 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:41.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:41.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:41.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:41.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:41.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:42.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:42.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 74012 73996 118.531 77.3125 0.055258 0.00843012 2024-08-17T17:40:42.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:42.154 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:42.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:42.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:42 smithi175 ceph-mon[101434]: pgmap v1176: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-08-17T17:40:42.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:42 smithi064 ceph-mon[104516]: pgmap v1176: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-08-17T17:40:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:42 smithi064 ceph-mon[109246]: pgmap v1176: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-08-17T17:40:42.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:42.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:42.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:42.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:42.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:42.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:42.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:42.958 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:42.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:43.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:40:43.018895+0000 min lat: 0.0012219 max lat: 0.230089 avg lat: 0.00852339 2024-08-17T17:40:43.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:40:43.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 75086 75070 117.246 67.125 0.0221561 0.00852339 2024-08-17T17:40:43.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:43.117 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:43.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:43.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:43.321 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:43.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:43.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:43.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:43.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:43.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:43.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:43.745 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:43.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:43.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:43.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:44.018 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 76313 76297 116.257 76.6875 0.00437688 0.00859593 2024-08-17T17:40:44.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:44.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:44.156 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:44.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:44.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:44.295 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:43] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:40:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:44.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:44.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:44.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:44.524 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:44.524 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:44.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:44.571 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:44.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:44.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:44 smithi064 ceph-mon[109246]: pgmap v1177: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2024-08-17T17:40:44.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:44 smithi064 ceph-mon[104516]: pgmap v1177: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2024-08-17T17:40:44.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:44 smithi175 ceph-mon[101434]: pgmap v1177: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2024-08-17T17:40:44.674 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:44.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:44.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:44.833 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:44.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:44.972 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:45.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 77601 77585 115.405 80.5 0.00989387 0.00866027 2024-08-17T17:40:45.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:45.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:45.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:45.248 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:45.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:45.411 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:45.513 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:45.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:45.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:45.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:45.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:45.811 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:45.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:45.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:46.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 78766 78750 114.415 72.8125 0.00834157 0.00873471 2024-08-17T17:40:46.053 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:46.081 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:46.182 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:46.320 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.518 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:46.518 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:46.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:46.601 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:46 smithi175 ceph-mon[101434]: pgmap v1178: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:40:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1151209074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[104516]: pgmap v1178: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:40:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1151209074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[109246]: pgmap v1178: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:40:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1151209074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:40:46.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.736 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:46.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:47.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:47.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 79964 79948 113.517 74.875 0.0124134 0.0088043 2024-08-17T17:40:47.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:47.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:47.189 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:40:47.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:47.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:47.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:47.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:47.493 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:40:47.493 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:47 smithi064 ceph-mon[104516]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:40:47.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:40:47.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:47 smithi064 ceph-mon[109246]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:40:47.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:47.553 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:47.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:47.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:40:47.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:47 smithi175 ceph-mon[101434]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:40:47.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:47.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:47.825 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:47.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:47.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:47.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:48.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 81259 81243 112.793 80.9375 0.0118146 0.00886096 2024-08-17T17:40:48.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:48.103 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:48.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:48.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:48.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:48.347 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:48.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:48.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:48.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:48.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:48.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:48 smithi175 ceph-mon[101434]: pgmap v1180: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:40:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:48 smithi175 ceph-mon[101434]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:40:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:48 smithi064 ceph-mon[104516]: pgmap v1180: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:40:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:48 smithi064 ceph-mon[104516]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:40:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:48 smithi064 ceph-mon[109246]: pgmap v1180: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:40:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:48 smithi064 ceph-mon[109246]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:40:48.754 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:48.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:48.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:48.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:49.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 82531 82515 112.069 79.5 0.0227255 0.00891772 2024-08-17T17:40:49.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:49.073 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:40:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=cleanup t=2024-08-17T17:40:48.897741961Z level=info msg="Completed cleanup jobs" duration=10.871895ms 2024-08-17T17:40:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:40:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana.update.checker t=2024-08-17T17:40:49.026984028Z level=info msg="Update check succeeded" duration=61.83922ms 2024-08-17T17:40:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:40:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugins.update.checker t=2024-08-17T17:40:49.053655755Z level=info msg="Update check succeeded" duration=61.801953ms 2024-08-17T17:40:49.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:49.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:49.317 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:49.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:49.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:49.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:49.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:49.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:49.646 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:49.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:49.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.933 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:49.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:50.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 83821 83805 111.4 80.625 0.0061585 0.00897098 2024-08-17T17:40:50.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:50.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:50.181 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:50.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:50.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:50.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:50.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.497 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:50.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:50.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:50.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:50 smithi064 ceph-mon[104516]: pgmap v1182: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:40:50.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:50 smithi064 ceph-mon[109246]: pgmap v1182: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:40:50.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:50.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:50.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:40:50.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:50 smithi175 ceph-mon[101434]: pgmap v1182: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:40:50.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:50.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.761 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:50.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:50.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:50.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:50.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:51.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 85004 84988 110.619 73.9375 0.0121357 0.00903366 2024-08-17T17:40:51.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:51.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:51.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:51.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:51.195 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:51.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:51.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:51.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:51.464 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:51.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:51.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:51.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:51.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:51.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:51.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:51.971 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:51.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:52.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 85981 85965 109.608 61.0625 0.0109013 0.00910202 2024-08-17T17:40:52.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:52.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:52.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:52.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:52.339 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:40:52.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:52.398 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:52.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:52.540 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:52.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:52 smithi064 ceph-mon[104516]: pgmap v1183: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:40:52.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:52 smithi064 ceph-mon[109246]: pgmap v1183: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:40:52.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:52.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:52 smithi175 ceph-mon[101434]: pgmap v1183: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:40:52.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:40:52.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:52.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:52.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:52.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:52.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:52.948 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:53.019 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 87117 87101 108.836 71 0.0261493 0.00918266 2024-08-17T17:40:53.050 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:53.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:53.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:53.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:53.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:53.422 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:40:53.423 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:53.456 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:40:53.457 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:53.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:53.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[104516]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:40:53.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-mon[109246]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:40:53.493 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:40:53.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:53.519 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:53.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:53.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:40:53.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:53.563 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:40:53.563 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:53.600 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:40:53.600 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:53.621 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:53.639 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:40:53.640 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:53.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:53.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:53 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3068715753' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:53 smithi175 ceph-mon[101434]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:40:53.678 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:40:53.678 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:53.757 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:53.785 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:53.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:53.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:53.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:54.020 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 88281 88265 108.129 72.75 0.00742169 0.00924326 2024-08-17T17:40:54.031 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:54.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:54.097 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:40:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:40:53] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.51.0" 2024-08-17T17:40:54.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.200 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:54.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:54.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:54.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.472 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.485 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:54.547 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:54.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:54.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:54.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:54 smithi064 ceph-mon[104516]: pgmap v1185: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:40:54.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1064817424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:54.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:54 smithi064 ceph-mon[109246]: pgmap v1185: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:40:54.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1064817424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:54 smithi175 ceph-mon[101434]: pgmap v1185: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:40:54.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1064817424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:54.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:54.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:54.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:54.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:54.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:55.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:55.022 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 89455 89439 107.46 73.375 0.00828666 0.00930085 2024-08-17T17:40:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:55.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:55.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:55.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:55.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:55.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:55.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:55.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:40:55.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:55.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:55.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:56.020 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 90642 90626 106.833 74.1875 0.0219298 0.0093549 2024-08-17T17:40:56.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:56.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:56.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:56.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:56.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:56 smithi175 ceph-mon[101434]: pgmap v1186: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:40:56.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[104516]: pgmap v1186: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:40:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:40:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:56 smithi064 ceph-mon[109246]: pgmap v1186: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:40:56.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:56.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:56.580 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:56.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:56.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:56.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:56.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:56.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.995 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:56.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:57.020 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 91719 91703 106.101 67.3125 0.0198513 0.00942021 2024-08-17T17:40:57.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:57.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:57.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:57.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:57.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:57.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:57.530 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:57.632 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:57.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:57.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:57.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:57.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:57.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:58.020 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 92856 92840 105.464 71.0625 0.00490414 0.00947688 2024-08-17T17:40:58.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.068 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:58.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.197 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:58.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:58.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:58.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:58.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:58.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:58.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:58.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:58.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:40:58 smithi175 ceph-mon[101434]: pgmap v1187: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:40:58.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:40:58 smithi064 ceph-mon[104516]: pgmap v1187: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:40:58.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:40:58 smithi064 ceph-mon[109246]: pgmap v1187: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:40:58.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:58.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:58.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:58.817 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:58.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:58.887 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:58.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:59.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 93871 93855 104.712 63.4375 0.0134618 0.00954422 2024-08-17T17:40:59.032 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:59.134 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:59.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.160 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:59.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:59.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:59.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:59.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:59.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:40:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:59.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:59.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:59.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:59.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:59.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:59.839 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:59.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:59.962 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:40:59.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 94914 94898 104.019 65.1875 0.00890298 0.00960877 2024-08-17T17:41:00.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:00.097 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:00.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:00.236 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:00.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:00.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:00.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:00.519 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:00.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:00.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:00 smithi064 ceph-mon[104516]: pgmap v1188: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:41:00.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:00 smithi064 ceph-mon[109246]: pgmap v1188: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:00 smithi175 ceph-mon[101434]: pgmap v1188: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:41:00.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.784 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:00.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:00.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:00.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 96088 96072 103.49 73.375 0.0151337 0.00965751 2024-08-17T17:41:01.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.061 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:01.193 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:01.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:01.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:01.428 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:01.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:01.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:01.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:01.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:01.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:01.796 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:41:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:41:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:41:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:41:01.796 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:41:01.829 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:01.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:01.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:01.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:02.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 97115 97099 102.824 64.1875 0.0168271 0.00972065 2024-08-17T17:41:02.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:02.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.227 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:02.264 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:02.329 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:41:02.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:02.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:02 smithi064 ceph-mon[104516]: pgmap v1189: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:41:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:41:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:02 smithi064 ceph-mon[109246]: pgmap v1189: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:41:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:41:02.411 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:02.513 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:02.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:02.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:02 smithi175 ceph-mon[101434]: pgmap v1189: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:41:02.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:41:02.678 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:02.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:02.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:02.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:02.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:41:03.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:41:03.021602+0000 min lat: 0.0012219 max lat: 0.230089 avg lat: 0.0097891 2024-08-17T17:41:03.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:41:03.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 98065 98049 102.1 59.375 0.00789229 0.0097891 2024-08-17T17:41:03.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:03.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:03.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:03.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.341 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:41:03.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:03.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:03.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:03.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:41:03.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:03 smithi175 ceph-mon[101434]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:41:03.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[104516]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/305883009' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[109246]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:41:03.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:03.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:41:03.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:03.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:03.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:03.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:03.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:03.977 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:04.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:04.025 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 98924 98908 101.306 53.6875 0.0315171 0.00986499 2024-08-17T17:41:04.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:04.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:04.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:04.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:04.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:04.325 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:03] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-08-17T17:41:04.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:04.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:04.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:04.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[104516]: pgmap v1191: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:41:04.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:04.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:04.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[104516]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:41:04.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[109246]: pgmap v1191: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:41:04.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:04.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:04.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:04 smithi064 ceph-mon[109246]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:41:04.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:04.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:04 smithi175 ceph-mon[101434]: pgmap v1191: 99 pgs: 99 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:41:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:04 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:04 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/72778139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:04 smithi175 ceph-mon[101434]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:41:04.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:04.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:04.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:04.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:05.022 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 99926 99910 100.682 62.625 0.0162243 0.00992653 2024-08-17T17:41:05.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:05.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:05.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:05.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:05.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:05.339 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:05 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.440 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:05.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:05.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:05.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:05.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:05 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:05.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:05.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:05.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.866 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:05.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:05.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:06.022 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 100847 100831 99.998 57.5625 0.00539221 0.00998598 2024-08-17T17:41:06.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:06.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:06.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:06.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:06.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:06.404 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:06.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:06.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:06.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:06.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:41:06.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:06 smithi175 ceph-mon[101434]: pgmap v1193: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-08-17T17:41:06.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:06 smithi064 ceph-mon[104516]: pgmap v1193: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-08-17T17:41:06.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:06 smithi064 ceph-mon[109246]: pgmap v1193: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-08-17T17:41:06.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.781 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:06.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:06.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:06.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.990 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:07.023 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 101878 101862 99.4404 64.4375 0.0125727 0.0100509 2024-08-17T17:41:07.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.099 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:07.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:07.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:07.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:07.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:07.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:07.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:07.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:07.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:07.661 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:07.763 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:07.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:07.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:07.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:08.023 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 102922 102906 98.9144 65.25 0.00797757 0.010104 2024-08-17T17:41:08.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.075 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:08.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:08.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:08.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:08.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:08.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:08.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:08 smithi064 ceph-mon[109246]: pgmap v1194: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:41:08.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:08 smithi064 ceph-mon[104516]: pgmap v1194: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:41:08.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:08.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:08.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:08.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:08.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:08 smithi175 ceph-mon[101434]: pgmap v1194: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:41:08.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:08.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:08.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:08.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:08.894 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:08.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.023 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 103927 103911 98.3674 62.8125 0.0118935 0.0101609 2024-08-17T17:41:09.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:41:09.138 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:09.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:09.199 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:09.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.472 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:09.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:09.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:09.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:09.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/345083792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:09.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/345083792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:09.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/345083792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:09.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:09.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:09.868 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:09.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:09.977 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:09.977 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:10.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.023 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 104941 104925 97.8449 63.375 0.0333325 0.0102143 2024-08-17T17:41:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.179 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:10.316 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:10.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:10.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:10.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:10 smithi175 ceph-mon[101434]: pgmap v1195: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:41:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:10 smithi064 ceph-mon[104516]: pgmap v1195: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:41:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:10 smithi064 ceph-mon[109246]: pgmap v1195: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:41:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:10.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:10.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:10.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:10.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:10.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:11.024 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 105989 105973 97.3693 65.5 0.0134028 0.0102653 2024-08-17T17:41:11.028 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:11.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.183 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:41:11.243 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:11.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:11.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:11.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:11.539 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:11.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:11.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:11.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.978 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:11.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:12.024 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 106920 106904 96.8015 58.1875 0.0159291 0.0103257 2024-08-17T17:41:12.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:12.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:12.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:12.247 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:12.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:12.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:12.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:12.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:41:12.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:12.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:12.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:12.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:12.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:12 smithi064 ceph-mon[104516]: pgmap v1196: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:12.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:12 smithi064 ceph-mon[109246]: pgmap v1196: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:12.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.793 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:12.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:12.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:12.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:12 smithi175 ceph-mon[101434]: pgmap v1196: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:12.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.024 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 107908 107892 96.3008 61.75 0.0129656 0.010379 2024-08-17T17:41:13.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:13.068 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:13.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:13.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:13.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:13.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:13.862 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:13.896 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:13.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:14.024 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:14.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 108936 108920 95.8472 64.25 0.0107884 0.0104273 2024-08-17T17:41:14.126 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:14.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:14.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:14.311 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:13] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.51.0" 2024-08-17T17:41:14.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:14.416 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:14.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:14.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:14.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:14.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:14 smithi064 ceph-mon[104516]: pgmap v1197: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:14.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:14 smithi064 ceph-mon[109246]: pgmap v1197: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:14.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:14.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:14.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:14.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:14.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:14 smithi175 ceph-mon[101434]: pgmap v1197: 99 pgs: 99 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:14.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:14.988 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:15.025 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 109873 109857 95.3294 58.5625 0.0166574 0.0104844 2024-08-17T17:41:15.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:15.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:15.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:15.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:15.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.513 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:15.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:15.640 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:15.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:15.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:15.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:15.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:15.806 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:15.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:15.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:16.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 110750 110734 94.7744 54.8125 0.00854043 0.0105463 2024-08-17T17:41:16.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:16.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:16.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:16.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:16.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.378 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:16.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:16.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.511 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:16.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:16.522 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:41:16.525 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:41:16.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:16.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:16.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:16.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:16.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:16.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[109246]: pgmap v1198: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T17:41:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2529676337' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3502422034' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[104516]: pgmap v1198: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T17:41:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2529676337' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3502422034' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:16.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:16 smithi175 ceph-mon[101434]: pgmap v1198: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T17:41:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2529676337' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:41:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3502422034' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:16.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:16.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:17.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 15 111692 111677 94.2902 58.9375 0.0280409 0.0106003 2024-08-17T17:41:17.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:17.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:17.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:17.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:17.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:17.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:17.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:17.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:17.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[109246]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:17.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:17 smithi064 ceph-mon[104516]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:41:17.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:17.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:17.833 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:17.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:17.869 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:17.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:17 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:17.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:41:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:17 smithi175 ceph-mon[101434]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:41:17.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.010 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:18.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:18.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 112720 112704 93.8888 64.1875 0.0250982 0.0106461 2024-08-17T17:41:18.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:18.143 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:18.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:18.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:18.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.422 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:18.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:18.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:18.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:18.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:18.694 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:18.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:18.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:18 smithi175 ceph-mon[101434]: pgmap v1200: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:18.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:18 smithi064 ceph-mon[109246]: pgmap v1200: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:18 smithi064 ceph-mon[104516]: pgmap v1200: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:41:18.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:19.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 113739 113723 93.4915 63.6875 0.0201872 0.0106903 2024-08-17T17:41:19.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:19.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:19.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:19.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:19.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:19.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:19.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:19.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:19.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:19.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:19.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:19.757 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:19.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:19.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:19.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:20.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 114767 114751 93.1117 64.25 0.0209285 0.0107347 2024-08-17T17:41:20.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:20.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:20.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:20.187 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:20.288 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:20.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:20.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.455 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:20.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:20.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:20.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:20.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:20.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:20 smithi064 ceph-mon[109246]: pgmap v1201: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T17:41:20.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:20 smithi064 ceph-mon[104516]: pgmap v1201: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T17:41:20.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:20 smithi175 ceph-mon[101434]: pgmap v1201: 99 pgs: 99 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T17:41:20.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.981 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:20.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:21.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 115599 115583 92.5847 52 0.0162336 0.0107956 2024-08-17T17:41:21.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:21.116 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:21.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:21.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:21.258 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:21.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:21.394 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:21.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.541 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:21.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:21.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.685 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:21.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:21.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:21.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:41:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.838 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:21.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:21.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:21.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:22.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 116601 116585 92.2055 62.625 0.012532 0.0108402 2024-08-17T17:41:22.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:22.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:22.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:22.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:22.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:22.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:22.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.528 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:22.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:22.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:22.638 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:22.638 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:22.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:22 smithi064 ceph-mon[104516]: pgmap v1202: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-17T17:41:22.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:22 smithi064 ceph-mon[109246]: pgmap v1202: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-17T17:41:22.657 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:22.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:22.791 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:22.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:41:22.892 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:22 smithi175 ceph-mon[101434]: pgmap v1202: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-17T17:41:22.919 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:23.021 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:23.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:41:23.026796+0000 min lat: 0.0012219 max lat: 0.230089 avg lat: 0.0108963 2024-08-17T17:41:23.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:41:23.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 117463 117447 91.7264 53.875 0.0307235 0.0108963 2024-08-17T17:41:23.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:23.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:23.193 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:23.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:23.349 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:23.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:23.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:23.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:23.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:23.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:23.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:23.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:23.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:23.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:23.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:23.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:24.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 118465 118449 91.3671 62.625 0.0208398 0.0109395 2024-08-17T17:41:24.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:24.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:41:24.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:24.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:24.277 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:23] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.51.0" 2024-08-17T17:41:24.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:24.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:24.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:24.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:24.588 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:24.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:24.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:24.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:24.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:24.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:24 smithi064 ceph-mon[104516]: pgmap v1203: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T17:41:24.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:24 smithi064 ceph-mon[109246]: pgmap v1203: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T17:41:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:24 smithi175 ceph-mon[101434]: pgmap v1203: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T17:41:24.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:24.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:25.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 119380 119364 90.9503 57.1875 0.0169802 0.0109897 2024-08-17T17:41:25.075 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:25.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:25.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:25.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:25.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:25.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:25.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:25.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:25.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:25.622 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:25.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:25.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:25.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:25.778 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:25.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:25.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:25.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:25.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:25.938 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:25.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:41:25.964 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:25.997 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:41:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:26.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 120304 120288 90.5504 57.75 0.0229955 0.011038 2024-08-17T17:41:26.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:41:26.029 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:26.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:26.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:41:26.066 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:26.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:26.104 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:41:26.105 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:26.141 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:41:26.142 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:26.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.175 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:41:26.175 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:26.210 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:41:26.210 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:26.215 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:26.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.357 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:26.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:26.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:26.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:26.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:26.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:26.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:26.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:26.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.876 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:26.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:26 smithi064 ceph-mon[104516]: pgmap v1204: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-08-17T17:41:26.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:26 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2208466429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:26.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:26 smithi064 ceph-mon[109246]: pgmap v1204: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-08-17T17:41:26.879 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:26 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2208466429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:26 smithi175 ceph-mon[101434]: pgmap v1204: 99 pgs: 99 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-08-17T17:41:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:26 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2208466429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:27.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 121244 121228 90.1705 58.75 0.0208152 0.011085 2024-08-17T17:41:27.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:27.063 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:27.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:27.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:27.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:27.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:27.344 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:27.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:27.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:27.479 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:27.581 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:27.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:27.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:27.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:27.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:27.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:27.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:28.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 122206 122190 89.817 60.125 0.0231767 0.0111283 2024-08-17T17:41:28.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:28.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:28.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:28.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:28.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:28.290 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:28.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.429 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:28.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:28.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:28.570 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:28.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:28.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:28.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:28.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:28.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:28.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:28 smithi064 ceph-mon[104516]: pgmap v1205: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:41:28.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:28 smithi064 ceph-mon[109246]: pgmap v1205: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:41:28.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:28 smithi175 ceph-mon[101434]: pgmap v1205: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:41:28.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:28.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:29.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 123145 123129 89.4551 58.6875 0.0262635 0.0111738 2024-08-17T17:41:29.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:29.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:29.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:29.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:29.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:29.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:29.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:29.553 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:29.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:29.654 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.682 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:29.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:29.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:29.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:29.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:29.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:30.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 124068 124052 89.09 57.6875 0.0239903 0.0112198 2024-08-17T17:41:30.057 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:30.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:30.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:30.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:30.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:30.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:30.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:30.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:30.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:30.591 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:41:30.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:30.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:30.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:30.772 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:30.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:30.895 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:30.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:30.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:30 smithi175 ceph-mon[101434]: pgmap v1206: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:41:30.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2923644497' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[104516]: pgmap v1206: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:41:30.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2923644497' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[109246]: pgmap v1206: 99 pgs: 99 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:41:30.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2923644497' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:41:30.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:31.020 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:31.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 124863 124847 88.6423 49.6875 0.00367233 0.0112426 2024-08-17T17:41:31.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:31.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:31.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:31.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:31.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:31.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:31.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:31.392 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:31.494 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:31.515 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:31.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:31.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:31.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:31.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:31.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:31.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:31.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:31.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:31 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:41:31.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:31 smithi175 ceph-mon[101434]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:41:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:31 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:41:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:31 smithi064 ceph-mon[104516]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:41:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:31 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:41:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:31 smithi064 ceph-mon[109246]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:41:31.989 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:32.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:32.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 124863 124847 87.6465 0 - 0.0112426 2024-08-17T17:41:32.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:32.137 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:32.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:32.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:32.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:32.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:32.490 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:32.514 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:32.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:32.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:32.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:32.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:32.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:32.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:32.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:32 smithi175 ceph-mon[101434]: pgmap v1208: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:32 smithi175 ceph-mon[101434]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:41:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:32 smithi064 ceph-mon[104516]: pgmap v1208: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:32 smithi064 ceph-mon[104516]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:41:32.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:32 smithi064 ceph-mon[109246]: pgmap v1208: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:41:32.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:32 smithi064 ceph-mon[109246]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:41:33.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:33.025 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:33.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 90 6 124992 124986 86.7694 4.34375 0.076545 0.0115174 2024-08-17T17:41:33.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:33.149 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:33.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:33.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:33.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:33.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:33.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:33.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:33.648 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:33.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:33.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:33.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:33.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:33.890 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:33.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:33.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:33 smithi175 ceph-mon[101434]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:41:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:33 smithi175 ceph-mon[101434]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:41:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2897643923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[104516]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:41:33.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[104516]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:41:33.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2897643923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[109246]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:41:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[109246]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:41:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2897643923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:33.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:34.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:34.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0547 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 124992 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 86.7473 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 39.0534 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 244.438 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:41:34.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1387 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 624.866 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3911 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0115195 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0258645 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.24721 2024-08-17T17:41:34.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.0012219 2024-08-17T17:41:34.035 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:41:34.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:34.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:34.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:34.268 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:33] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-08-17T17:41:34.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:34.376 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:41:34.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:34.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:34.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:34.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.587 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:34.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:34.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:34.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:34.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:41:34.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:34.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:34.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:34 smithi175 ceph-mon[101434]: pgmap v1211: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-08-17T17:41:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:34 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:34 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:41:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:34 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:34 smithi175 ceph-mon[101434]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:41:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[104516]: pgmap v1211: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-08-17T17:41:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:41:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[104516]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[109246]: pgmap v1211: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:34 smithi064 ceph-mon[109246]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:41:34.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:34.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:35.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:35.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:35.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:35.272 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:35.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:35.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:35.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:35.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:35.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.532 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:35.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:35.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:35.669 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:35.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:35.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:35.904 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:35.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:36.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:36.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:36.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.214 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:36.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:36.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:36.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:36.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:36.449 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:36.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:36.576 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:36.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:36.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:36.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:36.840 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:36.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:36.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:36 smithi175 ceph-mon[101434]: pgmap v1214: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.7 MiB/s wr, 115 op/s 2024-08-17T17:41:36.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:36 smithi064 ceph-mon[104516]: pgmap v1214: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.7 MiB/s wr, 115 op/s 2024-08-17T17:41:36.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:36 smithi064 ceph-mon[109246]: pgmap v1214: 99 pgs: 1 peering, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.7 MiB/s wr, 115 op/s 2024-08-17T17:41:36.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:37.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:37.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.208 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:37.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:37.222 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:41:37.222 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:41:37.222 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:41:37.222 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:41:37.222 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:41:37.310 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:37.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:37.445 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:37.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:37.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:37.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:37.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:37.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:37.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:37.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:37.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:37 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 2/371697 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:37.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:37 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 2/371697 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:37.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:38.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:38.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:37 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 2/371697 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:38.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:38.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:38.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:38.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:41:38.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:38.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:38.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:41:38.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:41:38.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:41:38.463 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:38.463 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:38.463 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:38.484 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:38.486 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:41:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:38.516 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:41:38.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:38.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:38.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:41:38.554 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:38.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:38.589 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:41:38.589 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:38.623 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:41:38.624 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:38.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:38.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:38.812 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:38.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:38 smithi064 ceph-mon[104516]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 180 B/s rd, 7.6 MiB/s wr, 319 op/s; 2/371697 objects degraded (0.001%); 7738/371697 objects misplaced (2.082%); 667 KiB/s, 12 objects/s recovering 2024-08-17T17:41:38.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:38 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2779627368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:38.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:38 smithi064 ceph-mon[109246]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 180 B/s rd, 7.6 MiB/s wr, 319 op/s; 2/371697 objects degraded (0.001%); 7738/371697 objects misplaced (2.082%); 667 KiB/s, 12 objects/s recovering 2024-08-17T17:41:38.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:38 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2779627368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:38.947 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:38.971 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:38.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:41:39.072 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:39.096 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:38 smithi175 ceph-mon[101434]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 180 B/s rd, 7.6 MiB/s wr, 319 op/s; 2/371697 objects degraded (0.001%); 7738/371697 objects misplaced (2.082%); 667 KiB/s, 12 objects/s recovering 2024-08-17T17:41:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:38 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2779627368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:39.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:39.236 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:39.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:39.368 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:39.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:39.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:39.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:39.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:39.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:39.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:39.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:39.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:39.911 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:40.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:40.049 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:40.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:40.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:40.283 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:40.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:40.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:40.447 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:40.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:40.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:40.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:40.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:40.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:40.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:40.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[104516]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 153 B/s rd, 6.4 MiB/s wr, 430 op/s; 2/368526 objects degraded (0.001%); 7738/368526 objects misplaced (2.100%); 567 KiB/s, 10 objects/s recovering 2024-08-17T17:41:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:40.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[109246]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 153 B/s rd, 6.4 MiB/s wr, 430 op/s; 2/368526 objects degraded (0.001%); 7738/368526 objects misplaced (2.100%); 567 KiB/s, 10 objects/s recovering 2024-08-17T17:41:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:40.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:40.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:41.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:41.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:41.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:40 smithi175 ceph-mon[101434]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 153 B/s rd, 6.4 MiB/s wr, 430 op/s; 2/368526 objects degraded (0.001%); 7738/368526 objects misplaced (2.100%); 567 KiB/s, 10 objects/s recovering 2024-08-17T17:41:41.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:41.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:41.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:41.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:41.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:41.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:41.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.529 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:41.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:41.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:41.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:41.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:41.948 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:42.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:42.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:42.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:42.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:42.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:42.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:42.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:42.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:42.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:42.614 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:42.617 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:42.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:42.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:42.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:42 smithi064 ceph-mon[104516]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 5.4 MiB/s wr, 752 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 928 KiB/s, 15 objects/s recovering 2024-08-17T17:41:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:42 smithi064 ceph-mon[109246]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 5.4 MiB/s wr, 752 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 928 KiB/s, 15 objects/s recovering 2024-08-17T17:41:42.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:42.885 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:42.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:42.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:43.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:43.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:43.157 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:43.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:42 smithi175 ceph-mon[101434]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 5.4 MiB/s wr, 752 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 928 KiB/s, 15 objects/s recovering 2024-08-17T17:41:43.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:43.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:43.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:43.420 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:43.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:43.553 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:43.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:43.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.821 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:43.922 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 198/359070 objects degraded (0.055%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 198/359070 objects degraded (0.055%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:43.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:43.954 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:44.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:44.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:44.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 198/359070 objects degraded (0.055%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:44.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:44.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:44.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:44.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:44.310 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:43] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-08-17T17:41:44.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:44.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:44.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:44.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:44.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:44.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:41:44.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:44.639 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:44.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:44.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:44.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:44.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:44 smithi064 ceph-mon[104516]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 4.3 MiB/s wr, 607 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 749 KiB/s, 12 objects/s recovering 2024-08-17T17:41:44.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:44 smithi064 ceph-mon[109246]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 4.3 MiB/s wr, 607 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 749 KiB/s, 12 objects/s recovering 2024-08-17T17:41:44.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:44.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:45.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:45.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:45.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:44 smithi175 ceph-mon[101434]: pgmap v1218: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 4.3 MiB/s wr, 607 op/s; 198/359070 objects degraded (0.055%); 7485/359070 objects misplaced (2.085%); 749 KiB/s, 12 objects/s recovering 2024-08-17T17:41:45.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:45.258 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:45.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:45.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:45.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:45.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:45.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:45.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:45.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:45.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:45.796 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:45.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:45.936 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:46.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:46.044 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:46.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:46.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:46.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:46.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:46.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:46.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:46.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:41:46.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:46.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:46.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:46.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:46.370 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:46.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:46.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:46.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:46.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:46.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:46.714 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:41:46.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:46.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:46.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[104516]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 607 op/s; 198/355881 objects degraded (0.056%); 7485/355881 objects misplaced (2.103%); 681 KiB/s, 11 objects/s recovering 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3333369883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[109246]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 607 op/s; 198/355881 objects degraded (0.056%); 7485/355881 objects misplaced (2.103%); 681 KiB/s, 11 objects/s recovering 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3333369883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:46.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:46.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:46.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:47.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:47.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:47.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:46 smithi175 ceph-mon[101434]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 93 B/s rd, 1.5 MiB/s wr, 607 op/s; 198/355881 objects degraded (0.056%); 7485/355881 objects misplaced (2.103%); 681 KiB/s, 11 objects/s recovering 2024-08-17T17:41:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3333369883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:41:47.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:47.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:47.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:47.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:47.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:47.488 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:47.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:47.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:47.726 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:47.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:47.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:47.896 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:47.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:41:47.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:47 smithi064 ceph-mon[104516]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:41:47.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:41:47.899 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:47 smithi064 ceph-mon[109246]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:41:47.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:48.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:48.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:48.206 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:41:48.206 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:47 smithi175 ceph-mon[101434]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:41:48.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:48.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:48.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:41:48.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:48.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:48.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:48.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:48.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:48.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:48.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:48.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.927 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:48.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:48.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:48 smithi064 ceph-mon[104516]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 419/346143 objects degraded (0.121%); 7214/346143 objects misplaced (2.084%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:48.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 419/346143 objects degraded (0.121%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:48 smithi064 ceph-mon[109246]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 419/346143 objects degraded (0.121%); 7214/346143 objects misplaced (2.084%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 419/346143 objects degraded (0.121%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:49.029 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:49.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:49.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:49.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:48 smithi175 ceph-mon[101434]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 419/346143 objects degraded (0.121%); 7214/346143 objects misplaced (2.084%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:49.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 419/346143 objects degraded (0.121%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:49.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:49.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:49.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:49.462 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:49.490 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:49.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:49.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:49.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:49.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:49.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:49.862 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:49.862 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:49.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:49.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:50.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:50.039 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:50.140 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:50.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:50.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:50.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:50.304 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:50.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:50.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:50.547 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:50.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:50.691 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:50.724 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:50.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:50.864 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:50.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:50.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:50.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:51.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:50 smithi064 ceph-mon[104516]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 419/342987 objects degraded (0.122%); 7214/342987 objects misplaced (2.103%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:51.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:50 smithi064 ceph-mon[109246]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 419/342987 objects degraded (0.122%); 7214/342987 objects misplaced (2.103%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:51.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:51.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:51.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:50 smithi175 ceph-mon[101434]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 419/342987 objects degraded (0.122%); 7214/342987 objects misplaced (2.103%); 684 KiB/s, 10 objects/s recovering 2024-08-17T17:41:51.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:51.237 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:41:51.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:51.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:51.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:51.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:51.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:51.596 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:51.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:51.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:51.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:51.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:51.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.959 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:51.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:52.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:52.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:52.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:52.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:52.330 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:52.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:52.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:52.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:52.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:52.546 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:41:52.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:52.586 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:41:52.587 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:52.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:52.624 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:41:52.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:52.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:52.682 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:41:52.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:52.716 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:41:52.716 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:52.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:41:52.746 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:52.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:52.781 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:41:52.781 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:52.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:52.821 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:41:52.821 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:52 smithi064 ceph-mon[104516]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2865120530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:52 smithi064 ceph-mon[109246]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2865120530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:52.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:52.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:53.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:53.057 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:53.097 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:52 smithi175 ceph-mon[101434]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:53.097 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2865120530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:53.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:53.193 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:53.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:53.319 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:53.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:53.556 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:53.567 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:41:53.567 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:41:53.568 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:41:53.568 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:41:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:53.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:53.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:53.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:53.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:53.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:53.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:54.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 639/333111 objects degraded (0.192%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:54.071 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 639/333111 objects degraded (0.192%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:54.071 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:41:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:41:53] "GET /metrics HTTP/1.1" 200 36655 "" "Prometheus/2.51.0" 2024-08-17T17:41:54.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:54.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 639/333111 objects degraded (0.192%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:54.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:54.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:54.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:54.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:54.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:54.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:54.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:54.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:54.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:54.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:54.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:54.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:54.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:55.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:55.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:55.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:55.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:54 smithi175 ceph-mon[101434]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:55.167 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:55.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:54 smithi064 ceph-mon[104516]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:55.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:54 smithi064 ceph-mon[109246]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 639/333111 objects degraded (0.192%); 6944/333111 objects misplaced (2.085%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:55.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:55.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:55.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:55.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:41:55.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:55.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:55.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:55.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:55.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:55.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:55.774 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:55.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:55.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:55.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:56.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:56.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:56.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:56.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:56.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:56.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:56.413 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:56.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:56.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:56.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:56.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:56.835 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:56.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:56.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:57.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:57.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:57.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:56 smithi064 ceph-mon[104516]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 639/330057 objects degraded (0.194%); 6944/330057 objects misplaced (2.104%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:57.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:56 smithi064 ceph-mon[109246]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 639/330057 objects degraded (0.194%); 6944/330057 objects misplaced (2.104%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:57.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:56 smithi175 ceph-mon[101434]: pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 639/330057 objects degraded (0.194%); 6944/330057 objects misplaced (2.104%); 640 KiB/s, 9 objects/s recovering 2024-08-17T17:41:57.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:57.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:57.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:57.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:57.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:57.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:57.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:57.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:57.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:57.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:57.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:57.955 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:58.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.101 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:58.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:58.202 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:58.240 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:58.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:58.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:58.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:58.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:58.369 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:41:58.369 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:41:58.369 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:41:58.369 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:41:58.369 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:41:58.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:58.510 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:58.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:58.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:58.746 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:58.769 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:58.850 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:41:58.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:58.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:59.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:59.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:59.130 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:58 smithi175 ceph-mon[101434]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 853/321219 objects degraded (0.266%); 6674/321219 objects misplaced (2.078%); 642 KiB/s, 10 objects/s recovering 2024-08-17T17:41:59.130 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 853/321219 objects degraded (0.266%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:59.130 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:58 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/502591746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.130 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:58 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:59.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:59.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[104516]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 853/321219 objects degraded (0.266%); 6674/321219 objects misplaced (2.078%); 642 KiB/s, 10 objects/s recovering 2024-08-17T17:41:59.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 853/321219 objects degraded (0.266%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:59.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/502591746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[109246]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 853/321219 objects degraded (0.266%); 6674/321219 objects misplaced (2.078%); 642 KiB/s, 10 objects/s recovering 2024-08-17T17:41:59.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 853/321219 objects degraded (0.266%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:41:59.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/502591746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:58 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:59.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:59.318 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:59.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:59.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:59.571 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:59.575 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:41:59.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:59.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:59.706 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:59.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:41:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:59.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:59.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:59.929 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:41:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:59.961 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:00.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:00.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:00.044 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:00.044 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[104516]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:42:00.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:00.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:00.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:41:59 smithi064 ceph-mon[109246]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:42:00.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:00.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:59 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:00.102 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:59 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:00.103 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:41:59 smithi175 ceph-mon[101434]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:42:00.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:00.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:00.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:00.405 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:00.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:00.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.583 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:00.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:00.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:00.802 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:42:00.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:00.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:00.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:00.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:01.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:01.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:01.138 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:01.140 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:00 smithi175 ceph-mon[101434]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 853/318405 objects degraded (0.268%); 6674/318405 objects misplaced (2.096%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:42:01.140 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1099859694' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.140 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[104516]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 853/318405 objects degraded (0.268%); 6674/318405 objects misplaced (2.096%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:42:01.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1099859694' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[109246]: pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 853/318405 objects degraded (0.268%); 6674/318405 objects misplaced (2.096%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:42:01.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1099859694' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:01.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:01.275 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:01.376 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:01.416 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:01.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:01.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:01.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:01.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:01.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:01.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:01.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:01.830 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:01.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.944 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167789 7.1 7:899719dc:::benchmark_data_smithi064_210574_object68980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.747316+0000", 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.19359791700003, 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.96752279100000005, 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.945 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167789 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747316+0000", 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747316+0000", 2024-08-17T17:42:01.946 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747314+0000", 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747321+0000", 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0709999999999997e-06 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.947 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747336+0000", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.556e-05 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747345+0000", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2390000000000002e-06 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747407+0000", 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2843999999999997e-05 2024-08-17T17:42:01.948 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.747421+0000", 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3536999999999999e-05 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.705121+0000", 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.957699996 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.949 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.705202+0000", 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1328999999999993e-05 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.705295+0000", 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3137e-05 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.950 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.708470+0000", 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0031742739999999999 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.711417+0000", 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002947206 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714776+0000", 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003359131 2024-08-17T17:42:01.951 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714807+0000", 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0824999999999997e-05 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714838+0000", 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1547999999999997e-05 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.952 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167808 7.1 7:8a2f447d:::benchmark_data_smithi064_210574_object68999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.778104+0000", 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.1628091, 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.936880296, 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167808 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.953 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778104+0000", 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778104+0000", 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.954 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778103+0000", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778110+0000", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0280000000000003e-06 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778128+0000", 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7606999999999999e-05 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.955 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778135+0000", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3690000000000001e-06 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778191+0000", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6020999999999998e-05 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.778204+0000", 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3124e-05 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.956 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.708500+0000", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93029641799999996 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.708603+0000", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000102178 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.708671+0000", 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8250000000000006e-05 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.957 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714909+0000", 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0062383389999999999 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714935+0000", 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5989000000000001e-05 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.958 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714943+0000", 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5410000000000003e-06 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714967+0000", 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3920000000000001e-05 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.959 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714985+0000", 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7999000000000002e-05 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.960 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167821 7.1 7:853033a3:::benchmark_data_smithi064_210574_object69012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.797020+0000", 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.14389328899995, 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.91807046000000003, 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167821 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.961 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797020+0000", 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797020+0000", 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.962 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797019+0000", 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797025+0000", 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5879999999999997e-06 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797035+0000", 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0732e-05 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.963 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797041+0000", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7899999999999996e-06 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797090+0000", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8607999999999999e-05 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.797100+0000", 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0719e-05 2024-08-17T17:42:01.964 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.711447+0000", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.91434664499999996 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.711537+0000", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9574999999999995e-05 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.711611+0000", 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3953000000000002e-05 2024-08-17T17:42:01.965 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.714991+0000", 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003380755 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715024+0000", 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2626999999999998e-05 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.966 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715052+0000", 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7895999999999999e-05 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715074+0000", 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1934e-05 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715091+0000", 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6796999999999999e-05 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.967 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167857 7.1 7:86b6fb5c:::benchmark_data_smithi064_210574_object69048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.844870+0000", 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.09604350400002, 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88793853899999997, 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167857 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.968 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844870+0000", 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844870+0000", 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.969 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844868+0000", 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844880+0000", 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1666e-05 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.970 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844899+0000", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9599999999999999e-05 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844907+0000", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844968+0000", 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1118999999999995e-05 2024-08-17T17:42:01.971 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.844981+0000", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3189e-05 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715115+0000", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.87013384900000001 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715180+0000", 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5630999999999996e-05 2024-08-17T17:42:01.972 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.715467+0000", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000286626 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725691+0000", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01022349 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725699+0000", 2024-08-17T17:42:01.973 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1960000000000008e-06 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732747+0000", 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0070485440000000003 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732783+0000", 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5536e-05 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.974 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732809+0000", 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5660000000000002e-05 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167860 7.1 7:87b3ca5e:::benchmark_data_smithi064_210574_object69051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.850938+0000", 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.08997501500005, 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88198580199999999, 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.975 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167860 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850938+0000", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850938+0000", 2024-08-17T17:42:01.976 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850937+0000", 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850950+0000", 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3328e-05 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.977 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850968+0000", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8091999999999999e-05 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.850975+0000", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1790000000000002e-06 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851023+0000", 2024-08-17T17:42:01.978 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8084000000000002e-05 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851033+0000", 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.414e-06 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.718709+0000", 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.86767679799999997 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.979 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.718787+0000", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7173000000000001e-05 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.718883+0000", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6489000000000004e-05 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725730+0000", 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0068471319999999997 2024-08-17T17:42:01.980 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725766+0000", 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6019999999999997e-05 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732886+0000", 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.007119312 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.981 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732907+0000", 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1676e-05 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732924+0000", 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6977999999999999e-05 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.982 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167861 7.1 7:82958e26:::benchmark_data_smithi064_210574_object69052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.851330+0000", 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.08958350900002, 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88564461800000005, 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167861 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.983 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851330+0000", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851330+0000", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851329+0000", 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:01.984 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851335+0000", 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6919999999999999e-06 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851342+0000", 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.221e-06 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.985 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851346+0000", 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8319999999999999e-06 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851388+0000", 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1492999999999997e-05 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.851399+0000", 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1610999999999999e-05 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.986 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.722405+0000", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.871005578 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.722520+0000", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001154 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.722591+0000", 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1289000000000001e-05 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.987 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725759+0000", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0031674680000000001 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732696+0000", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0069375490000000003 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.736905+0000", 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042089880000000003 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.988 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.736935+0000", 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9447000000000001e-05 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.736975+0000", 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9666000000000003e-05 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.989 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.35217.0:167873 7.1 7:87f458c5:::benchmark_data_smithi064_210574_object69064:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:33:21.869526+0000", 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 520.071387706, 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88919166999999999, 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.35217", 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2066913824", 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 167873 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.990 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869526+0000", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869526+0000", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869524+0000", 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.991 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869536+0000", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2046e-05 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869555+0000", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8887999999999999e-05 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869564+0000", 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0149999999999992e-06 2024-08-17T17:42:01.992 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869610+0000", 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5618000000000003e-05 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:21.869619+0000", 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.166e-06 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.993 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725784+0000", 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85616535100000002 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725870+0000", 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5399e-05 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.994 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.725943+0000", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2939000000000001e-05 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732726+0000", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0067829850000000001 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.732815+0000", 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9339999999999995e-05 2024-08-17T17:42:01.995 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.758659+0000", 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.025844412000000001 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.758693+0000", 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3513999999999998e-05 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.996 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:33:22.758717+0000", 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4663000000000001e-05 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:01.997 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164887 8.2 8:47118bc0:::benchmark_data_smithi064_242541_object42394:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.038896+0000", 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 277.90201776700002, 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.87510359699999996, 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:01.998 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164887 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038896+0000", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038896+0000", 2024-08-17T17:42:01.999 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038892+0000", 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999962 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038910+0000", 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8563000000000002e-05 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.000 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038955+0000", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4790000000000003e-05 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038974+0000", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8544999999999999e-05 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.180886+0000", 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.14191230199999999 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.001 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.180895+0000", 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7080000000000003e-06 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.908107+0000", 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.72721193500000003 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:42:02.002 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.908280+0000", 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017290200000000001 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.913949+0000", 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0056694329999999998 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.913999+0000", 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0359999999999999e-05 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.003 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164894 8.12 8:4a73e350:::benchmark_data_smithi064_242541_object42401:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.038986+0000", 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 277.90192768600002, 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.90906878199999996, 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.004 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.006 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.006 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.006 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.006 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164894 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038986+0000", 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038986+0000", 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038985+0000", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038989+0000", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.371e-06 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.038995+0000", 2024-08-17T17:42:02.008 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7170000000000002e-06 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039002+0000", 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2960000000000004e-06 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.180912+0000", 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.14191071 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.009 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.180918+0000", 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7749999999999998e-06 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940404+0000", 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.75948550800000003 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.010 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940551+0000", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000146933 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.948003+0000", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0074523849999999997 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.948055+0000", 2024-08-17T17:42:02.011 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1576000000000002e-05 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:192645 8.14 8:2e9133ce:::benchmark_data_smithi064_242541_object70152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:01.757501+0000", 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 240.18341228, 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7322828079999999, 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.012 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 192645 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757501+0000", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757501+0000", 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.013 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757500+0000", 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757507+0000", 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.014 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757522+0000", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5588000000000001e-05 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757530+0000", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9030000000000004e-06 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757605+0000", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5098999999999994e-05 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.015 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.757615+0000", 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5880000000000004e-06 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330302+0000", 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.57268744999999999 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330307+0000", 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2339999999999996e-06 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.016 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.463081+0000", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.132774505 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.463176+0000", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5173000000000005e-05 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.463498+0000", 2024-08-17T17:42:02.017 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00032148300000000001 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.465710+0000", 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022119140000000002 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.471688+0000", 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0059777909999999997 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.018 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489011+0000", 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.017323051999999999 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489607+0000", 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00059613400000000001 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.019 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489683+0000", 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6276e-05 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489784+0000", 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101006 2024-08-17T17:42:02.020 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.021 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.021 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.021 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.021 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.021 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:192664 8.14 8:297cffa3:::benchmark_data_smithi064_242541_object70171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:01.794090+0000", 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 240.14682357199999, 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.695907611, 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.022 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 192664 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794090+0000", 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794090+0000", 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.023 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794089+0000", 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794094+0000", 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9849999999999997e-06 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.024 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794105+0000", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1484000000000001e-05 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794113+0000", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4370000000000001e-06 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794171+0000", 2024-08-17T17:42:02.025 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8613999999999998e-05 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.794182+0000", 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0557000000000001e-05 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330329+0000", 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.53614662199999996 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.026 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330333+0000", 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.014e-06 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.465746+0000", 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1354130389999999 2024-08-17T17:42:02.027 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.465815+0000", 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9696999999999994e-05 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.466166+0000", 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00035068099999999999 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.028 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.471799+0000", 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0056326199999999996 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.475464+0000", 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036652590000000001 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489054+0000", 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.013589921 2024-08-17T17:42:02.029 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489879+0000", 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00082531900000000003 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489916+0000", 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6795999999999998e-05 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.030 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489998+0000", 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1645999999999997e-05 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:192678 8.14 8:2985024c:::benchmark_data_smithi064_242541_object70185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:01.852836+0000", 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 240.08807773699999, 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6864250570000001, 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 192678 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852836+0000", 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.032 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852836+0000", 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852834+0000", 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.033 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852844+0000", 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0093e-05 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852868+0000", 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4369e-05 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852877+0000", 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2550000000000003e-06 2024-08-17T17:42:02.034 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852940+0000", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2533000000000006e-05 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:01.852955+0000", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.45e-05 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330346+0000", 2024-08-17T17:42:02.035 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.47739170600000003 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:02.330350+0000", 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9310000000000001e-06 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.468461+0000", 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.138111326 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.036 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.468586+0000", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000124156 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2,5", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.469054+0000", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00046873699999999999 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.475470+0000", 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0064158430000000001 2024-08-17T17:42:02.037 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.478357+0000", 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002886464 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.489351+0000", 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010993966000000001 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.038 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.539082+0000", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.049731466000000002 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.539143+0000", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0520000000000003e-05 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:03.539261+0000", 2024-08-17T17:42:02.039 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000118176 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198231 8.14 8:2c46667c:::benchmark_data_smithi064_242541_object75738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:10.366495+0000", 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 231.57441894499999, 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.97124143699999999, 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.040 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198231 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366495+0000", 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366495+0000", 2024-08-17T17:42:02.041 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366493+0000", 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366499+0000", 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.553e-06 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.042 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366510+0000", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1474000000000001e-05 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366518+0000", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2300000000000002e-06 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366577+0000", 2024-08-17T17:42:02.043 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8773999999999997e-05 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.366585+0000", 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8419999999999994e-06 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.330773+0000", 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.96418741500000005 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.044 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.330891+0000", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000118057 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.331225+0000", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000334679 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337481+0000", 2024-08-17T17:42:02.045 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0062557339999999998 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337510+0000", 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8331e-05 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337657+0000", 2024-08-17T17:42:02.046 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014711999999999999 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337702+0000", 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5383999999999998e-05 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337736+0000", 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3912999999999999e-05 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.047 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198250 8.14 8:295cd396:::benchmark_data_smithi064_242541_object75757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:10.383729+0000", 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 231.55718431, 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.95897010699999996, 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198250 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.048 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383729+0000", 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383729+0000", 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.049 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383727+0000", 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383735+0000", 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9400000000000002e-06 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383756+0000", 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0679999999999999e-05 2024-08-17T17:42:02.050 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383765+0000", 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7150000000000004e-06 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383816+0000", 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1121000000000001e-05 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.051 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.383836+0000", 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9831e-05 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.333198+0000", 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.94936274499999995 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.333272+0000", 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3942000000000005e-05 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.052 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.333444+0000", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017117 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337569+0000", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0041251320000000001 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.338027+0000", 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00045795800000000001 2024-08-17T17:42:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342614+0000", 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004587575 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342660+0000", 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6193999999999999e-05 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.054 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342699+0000", 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8838999999999999e-05 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198281 8.14 8:2e8d1518:::benchmark_data_smithi064_242541_object75788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:10.418487+0000", 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 231.52242648699999, 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.92431958199999997, 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.055 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198281 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418487+0000", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418487+0000", 2024-08-17T17:42:02.056 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418485+0000", 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418494+0000", 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9260000000000006e-06 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.057 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418512+0000", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7425999999999999e-05 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418521+0000", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6190000000000006e-06 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418580+0000", 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8491e-05 2024-08-17T17:42:02.058 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.418590+0000", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0318e-05 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.334534+0000", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.91594394499999998 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.334649+0000", 2024-08-17T17:42:02.059 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114632 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.334903+0000", 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025464200000000002 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337583+0000", 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026799879999999999 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.060 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.340492+0000", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002908898 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342742+0000", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022498869999999999 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342773+0000", 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1143999999999999e-05 2024-08-17T17:42:02.061 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342807+0000", 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.341e-05 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198344 8.14 8:2d847476:::benchmark_data_smithi064_242541_object75851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:10.523414+0000", 2024-08-17T17:42:02.062 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 231.41749950400001, 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81947365500000002, 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198344 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523414+0000", 2024-08-17T17:42:02.063 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523414+0000", 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523413+0000", 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.064 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523419+0000", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3219999999999998e-06 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523431+0000", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1712000000000001e-05 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523440+0000", 2024-08-17T17:42:02.065 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5779999999999999e-06 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523492+0000", 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2429000000000002e-05 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:10.523505+0000", 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3334e-05 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.066 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.336093+0000", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.81258794899999998 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.336159+0000", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5485000000000001e-05 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.336382+0000", 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00022299200000000001 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.067 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.337754+0000", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001372774 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.340602+0000", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00284732 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342843+0000", 2024-08-17T17:42:02.068 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002240953 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342868+0000", 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5123999999999999e-05 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:11.342888+0000", 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9762000000000002e-05 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.069 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198762 8.18 8.af8687b8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:13.455719+0000", 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 228.485194256, 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85823519400000003, 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.070 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198762 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455719+0000", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455719+0000", 2024-08-17T17:42:02.071 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455718+0000", 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455727+0000", 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0450000000000006e-06 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.072 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455741+0000", 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4864e-05 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455749+0000", 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3200000000000002e-06 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455819+0000", 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0068999999999996e-05 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.073 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455833+0000", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4524e-05 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:14.313945+0000", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85811153900000003 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:14.313954+0000", 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5219999999999996e-06 2024-08-17T17:42:02.074 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:198763 8.18 8.9e1180b8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:13.455762+0000", 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 228.485151645, 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85822100700000004, 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:02.075 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 198763 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455762+0000", 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455762+0000", 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.076 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455761+0000", 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455765+0000", 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.174e-06 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.077 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455773+0000", 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7299999999999994e-06 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455779+0000", 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.728e-06 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.078 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455853+0000", 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4233000000000006e-05 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:13.455858+0000", 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2530000000000004e-06 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.079 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:14.313975+0000", 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.85811679399999996 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:14.313983+0000", 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5809999999999999e-06 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.080 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45827.0:145106 9.c e397/375 9:31bb46a3:::benchmark_data_smithi064_275711_object20111:head v 397'9243, mlcod=397'9241)", 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:41:58.329691+0000", 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6112229810000001, 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.082781357999999999, 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329691+0000", 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.081 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329691+0000", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329689+0000", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329697+0000", 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7270000000000007e-06 2024-08-17T17:42:02.082 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329710+0000", 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3539e-05 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329718+0000", 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8290000000000008e-06 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:02.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:02.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329749+0000", 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0720999999999999e-05 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.329763+0000", 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4225e-05 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.085 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.412435+0000", 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.082671475999999994 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.412472+0000", 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7225999999999999e-05 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.086 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45827.0:145105 9.4 e397/375 9:26aba5ef:::benchmark_data_smithi064_275711_object20110:head v 397'9089, mlcod=397'9087)", 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:41:58.331091+0000", 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 3.6098221530000001, 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.084388629000000007, 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331091+0000", 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.087 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331091+0000", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331090+0000", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331098+0000", 2024-08-17T17:42:02.088 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5510000000000002e-06 2024-08-17T17:42:02.089 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.089 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.089 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:02.089 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331111+0000", 2024-08-17T17:42:02.089 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2856e-05 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331118+0000", 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8839999999999997e-06 2024-08-17T17:42:02.090 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331611+0000", 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00049317299999999996 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.091 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.331635+0000", 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4295999999999999e-05 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.415410+0000", 2024-08-17T17:42:02.092 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.083775217999999999 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:41:58.415480+0000", 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9665000000000005e-05 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.093 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.094 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:02.094 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:02.094 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:42:02.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:01 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:02.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:01 smithi064 ceph-mon[104516]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:42:02.134 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:01 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:02.134 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:01 smithi064 ceph-mon[109246]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:42:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:01 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:01 smithi175 ceph-mon[101434]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:42:02.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:02.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:02.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:02.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:02.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:02.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:02.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:02.571 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:42:02.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:02.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:02.678 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:02.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:02.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:02.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:02.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:02.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:42:03.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:03.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:03.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:03.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:02 smithi064 ceph-mon[104516]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:03.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:02 smithi064 ceph-mon[109246]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:02 smithi175 ceph-mon[101434]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:03.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:03.236 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:03.248 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:42:03.248 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:03.257 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:03.264 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:03.271 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:03.290 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:42:03.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:42:03.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:03.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:03.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:03.372 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:42:03.372 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:03.408 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:42:03.408 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:03.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:42:03.439 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:03.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:03.478 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:42:03.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:03.511 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:42:03.511 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.514 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:03.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:03.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:03.641 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:03.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:03.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:03.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:03.877 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:03.906 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:04.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:04.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:03 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2080062857' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1005/309990 objects degraded (0.324%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:04.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:03 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2080062857' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1005/309990 objects degraded (0.324%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:03] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.51.0" 2024-08-17T17:42:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:03 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2080062857' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1005/309990 objects degraded (0.324%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:04.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:04.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:04.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:04.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:04.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:04.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:04.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:04.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:04.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:04.729 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:04.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:04.871 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:04.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:05.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:05.055 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:04 smithi175 ceph-mon[101434]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:05.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:05.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:05.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:04 smithi064 ceph-mon[104516]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:05.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:04 smithi064 ceph-mon[109246]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1005/309990 objects degraded (0.324%); 6451/309990 objects misplaced (2.081%); 1016 KiB/s, 15 objects/s recovering 2024-08-17T17:42:05.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:05.283 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:05.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:05.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:05.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:05.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:05.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:05.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:05.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:05.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:05.733 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:05.733 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:05.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.832 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:05.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:05.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:05.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:06.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:06.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:06.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:06.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.254 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:06.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:06.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:06.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:06.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:42:06.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:42:06.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:06.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.555 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:06.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:06.656 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.696 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:06.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:06.749 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:06.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:06.875 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:06.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:06.987 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:06 smithi064 ceph-mon[104516]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1005/307062 objects degraded (0.327%); 6451/307062 objects misplaced (2.101%); 568 KiB/s, 8 objects/s recovering 2024-08-17T17:42:06.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:06 smithi064 ceph-mon[109246]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1005/307062 objects degraded (0.327%); 6451/307062 objects misplaced (2.101%); 568 KiB/s, 8 objects/s recovering 2024-08-17T17:42:07.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:07.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:07.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:06 smithi175 ceph-mon[101434]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1005/307062 objects degraded (0.327%); 6451/307062 objects misplaced (2.101%); 568 KiB/s, 8 objects/s recovering 2024-08-17T17:42:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:07.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:07.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:07.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:07.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:07.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:07.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.598 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:07.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:07.699 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:07.724 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:07.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:07.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:07.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:07.880 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:42:07.959 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:08.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:08.078 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:42:08.080 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:42:08.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:08.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:08.152 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/140680961' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.152 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:07 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/140680961' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:07 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/140680961' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:07 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:08.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.271 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:08.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:08.372 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:08.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:08.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:08.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:08.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:08.641 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:08.701 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:08.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:08.738 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:08.840 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:08.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:08.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.020 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:09.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:09.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:09.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:08 smithi175 ceph-mon[101434]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:42:09.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:08 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:09.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:08 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:09.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:08 smithi175 ceph-mon[101434]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:42:09.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1185/298188 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.164 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:09.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[104516]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:42:09.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:09.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:09.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[104516]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1185/298188 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[109246]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[109246]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:42:09.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1185/298188 objects degraded (0.397%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:09.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:09.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:09.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:09.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:09.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:42:09.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:09.568 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:09.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:09.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:09.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:09.828 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:09.832 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:09.933 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:09.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:10.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:10.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:09 smithi175 ceph-mon[101434]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:10.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:09 smithi064 ceph-mon[104516]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:10.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:09 smithi064 ceph-mon[109246]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:10.231 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:10.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:10.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:10.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:10.422 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:10.524 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.564 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:10.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:10.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:10.701 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:10.804 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:10.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:10.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:10.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:11.071 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1234,"stamp":"2024-08-17T17:42:07.877594+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6516905981,"num_objects":99396,"num_object_clones":0,"num_object_copies":298188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1185,"num_objects_misplaced":6218,"num_objects_unfound":0,"num_objects_dirty":99396,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":151526,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":517,"num_bytes_recovered":34347235,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129135,"ondisk_log_size":129135,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24821716,"kb_used_data":19279764,"kb_used_omap":46,"kb_used_meta":5541713,"kb_avail":725139500,"statfs":{"total":767960285184,"available":742542848000,"internally_reserved":0,"allocated":19742478336,"data_stored":19726835570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5674714355},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-441647104,"num_objects":-6739,"num_object_clones":0,"num_object_copies":-20217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":332,"num_objects_misplaced":-456,"num_objects_unfound":0,"num_objects_dirty":-6739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6739,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.606087"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":658,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.802649+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:42:00.802649+0000","last_peered":"2024-08-17T17:42:00.802649+0000","last_clean":"2024-08-17T17:42:00.802649+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:42:00.802649+0000","last_undegraded":"2024-08-17T17:42:00.802649+0000","last_fullsized":"2024-08-17T17:42:00.802649+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":399,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.783234+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:41:58.783234+0000","last_peered":"2024-08-17T17:41:58.783234+0000","last_clean":"2024-08-17T17:41:58.783234+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:41:58.783234+0000","last_undegraded":"2024-08-17T17:41:58.783234+0000","last_fullsized":"2024-08-17T17:41:58.783234+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":183,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780409+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:41:58.780409+0000","last_peered":"2024-08-17T17:41:58.780409+0000","last_clean":"2024-08-17T17:41:58.780409+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:41:58.780409+0000","last_undegraded":"2024-08-17T17:41:58.780409+0000","last_fullsized":"2024-08-17T17:41:58.780409+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":650,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808987+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:42:00.808987+0000","last_peered":"2024-08-17T17:42:00.808987+0000","last_clean":"2024-08-17T17:42:00.808987+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:42:00.808987+0000","last_undegraded":"2024-08-17T17:42:00.808987+0000","last_fullsized":"2024-08-17T17:42:00.808987+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":650,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811440+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:42:00.811440+0000","last_peered":"2024-08-17T17:42:00.811440+0000","last_clean":"2024-08-17T17:42:00.811440+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:42:00.811440+0000","last_undegraded":"2024-08-17T17:42:00.811440+0000","last_fullsized":"2024-08-17T17:42:00.811440+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":385,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.787025+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:41:58.787025+0000","last_peered":"2024-08-17T17:41:58.787025+0000","last_clean":"2024-08-17T17:41:58.787025+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:41:58.787025+0000","last_undegraded":"2024-08-17T17:41:58.787025+0000","last_fullsized":"2024-08-17T17:41:58.787025+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":658,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807990+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:42:00.807990+0000","last_peered":"2024-08-17T17:42:00.807990+0000","last_clean":"2024-08-17T17:42:00.807990+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:42:00.807990+0000","last_undegraded":"2024-08-17T17:42:00.807990+0000","last_fullsized":"2024-08-17T17:42:00.807990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":333,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.792702+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:41:58.792702+0000","last_peered":"2024-08-17T17:41:58.792702+0000","last_clean":"2024-08-17T17:41:58.792702+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:41:58.792702+0000","last_undegraded":"2024-08-17T17:41:58.792702+0000","last_fullsized":"2024-08-17T17:41:58.792702+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"399'9403","reported_seq":10255,"reported_epoch":399,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:05.600686+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:42:05.600686+0000","last_peered":"2024-08-17T17:42:05.600686+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:42:05.600686+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"380'300","ondisk_log_start":"380'300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9103,"log_dups_size":0,"ondisk_log_size":9103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409010176,"num_objects":6241,"num_object_clones":0,"num_object_copies":18723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1185,"num_objects_misplaced":6218,"num_objects_unfound":0,"num_objects_dirty":6241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9403,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":6218},{"shards":"0,5","objects":-1162}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":395,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.783596+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:41:58.783596+0000","last_peered":"2024-08-17T17:41:58.783596+0000","last_clean":"2024-08-17T17:41:58.783596+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:41:58.783596+0000","last_undegraded":"2024-08-17T17:41:58.783596+0000","last_fullsized":"2024-08-17T17:41:58.783596+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":194,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803127+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:42:00.803127+0000","last_peered":"2024-08-17T17:42:00.803127+0000","last_clean":"2024-08-17T17:42:00.803127+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:42:00.803127+0000","last_undegraded":"2024-08-17T17:42:00.803127+0000","last_fullsized":"2024-08-17T17:42:00.803127+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"399'9472","reported_seq":9532,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.250260+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:42:06.250260+0000","last_peered":"2024-08-17T17:42:06.250260+0000","last_clean":"2024-08-17T17:42:06.250260+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:42:06.250260+0000","last_undegraded":"2024-08-17T17:42:06.250260+0000","last_fullsized":"2024-08-17T17:42:06.250260+0000","mapping_epoch":375,"log_start":"382'2300","ondisk_log_start":"382'2300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404226048,"num_objects":6168,"num_object_clones":0,"num_object_copies":18504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9472,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":396,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.790674+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:41:58.790674+0000","last_peered":"2024-08-17T17:41:58.790674+0000","last_clean":"2024-08-17T17:41:58.790674+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:41:58.790674+0000","last_undegraded":"2024-08-17T17:41:58.790674+0000","last_fullsized":"2024-08-17T17:41:58.790674+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":291,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779658+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:41:58.779658+0000","last_peered":"2024-08-17T17:41:58.779658+0000","last_clean":"2024-08-17T17:41:58.779658+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:41:58.779658+0000","last_undegraded":"2024-08-17T17:41:58.779658+0000","last_fullsized":"2024-08-17T17:41:58.779658+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"399'9432","reported_seq":9466,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.018837+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:42:06.018837+0000","last_peered":"2024-08-17T17:42:06.018837+0000","last_clean":"2024-08-17T17:42:06.018837+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:42:06.018837+0000","last_undegraded":"2024-08-17T17:42:06.018837+0000","last_fullsized":"2024-08-17T17:42:06.018837+0000","mapping_epoch":375,"log_start":"380'600","ondisk_log_start":"380'600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8832,"log_dups_size":0,"ondisk_log_size":8832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406585344,"num_objects":6204,"num_object_clones":0,"num_object_copies":18612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9432,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":413,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.798150+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:41:58.798150+0000","last_peered":"2024-08-17T17:41:58.798150+0000","last_clean":"2024-08-17T17:41:58.798150+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:41:58.798150+0000","last_undegraded":"2024-08-17T17:41:58.798150+0000","last_fullsized":"2024-08-17T17:41:58.798150+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":313,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.789807+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:41:58.789807+0000","last_peered":"2024-08-17T17:41:58.789807+0000","last_clean":"2024-08-17T17:41:58.789807+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:41:58.789807+0000","last_undegraded":"2024-08-17T17:41:58.789807+0000","last_fullsized":"2024-08-17T17:41:58.789807+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"399'9503","reported_seq":9537,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.014843+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:42:06.014843+0000","last_peered":"2024-08-17T17:42:06.014843+0000","last_clean":"2024-08-17T17:42:06.014843+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:42:06.014843+0000","last_undegraded":"2024-08-17T17:42:06.014843+0000","last_fullsized":"2024-08-17T17:42:06.014843+0000","mapping_epoch":375,"log_start":"380'600","ondisk_log_start":"380'600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8903,"log_dups_size":0,"ondisk_log_size":8903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404160512,"num_objects":6167,"num_object_clones":0,"num_object_copies":18501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9503,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":193,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780006+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:41:58.780006+0000","last_peered":"2024-08-17T17:41:58.780006+0000","last_clean":"2024-08-17T17:41:58.780006+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:41:58.780006+0000","last_undegraded":"2024-08-17T17:41:58.780006+0000","last_fullsized":"2024-08-17T17:41:58.780006+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":665,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.810017+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:42:00.810017+0000","last_peered":"2024-08-17T17:42:00.810017+0000","last_clean":"2024-08-17T17:42:00.810017+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:42:00.810017+0000","last_undegraded":"2024-08-17T17:42:00.810017+0000","last_fullsized":"2024-08-17T17:42:00.810017+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"399'9397","reported_seq":9457,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:04.246694+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:42:04.246694+0000","last_peered":"2024-08-17T17:42:04.246694+0000","last_clean":"2024-08-17T17:42:04.246694+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:42:04.246694+0000","last_undegraded":"2024-08-17T17:42:04.246694+0000","last_fullsized":"2024-08-17T17:42:04.246694+0000","mapping_epoch":375,"log_start":"383'2400","ondisk_log_start":"383'2400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417398784,"num_objects":6369,"num_object_clones":0,"num_object_copies":19107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9397,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":754,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.786893+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:41:58.786893+0000","last_peered":"2024-08-17T17:41:58.786893+0000","last_clean":"2024-08-17T17:41:58.786893+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:41:58.786893+0000","last_undegraded":"2024-08-17T17:41:58.786893+0000","last_fullsized":"2024-08-17T17:41:58.786893+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":680,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.802919+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:42:00.802919+0000","last_peered":"2024-08-17T17:42:00.802919+0000","last_clean":"2024-08-17T17:42:00.802919+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:42:00.802919+0000","last_undegraded":"2024-08-17T17:42:00.802919+0000","last_fullsized":"2024-08-17T17:42:00.802919+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":314,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.782103+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:41:58.782103+0000","last_peered":"2024-08-17T17:41:58.782103+0000","last_clean":"2024-08-17T17:41:58.782103+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:41:58.782103+0000","last_undegraded":"2024-08-17T17:41:58.782103+0000","last_fullsized":"2024-08-17T17:41:58.782103+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"399'9213","reported_seq":9273,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.232862+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:42:06.232862+0000","last_peered":"2024-08-17T17:42:06.232862+0000","last_clean":"2024-08-17T17:42:06.232862+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:42:06.232862+0000","last_undegraded":"2024-08-17T17:42:06.232862+0000","last_fullsized":"2024-08-17T17:42:06.232862+0000","mapping_epoch":375,"log_start":"382'2000","ondisk_log_start":"382'2000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7213,"log_dups_size":0,"ondisk_log_size":7213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394854400,"num_objects":6025,"num_object_clones":0,"num_object_copies":18075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9213,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":415,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780624+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:41:58.780624+0000","last_peered":"2024-08-17T17:41:58.780624+0000","last_clean":"2024-08-17T17:41:58.780624+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:41:58.780624+0000","last_undegraded":"2024-08-17T17:41:58.780624+0000","last_fullsized":"2024-08-17T17:41:58.780624+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":320,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779346+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:41:58.779346+0000","last_peered":"2024-08-17T17:41:58.779346+0000","last_clean":"2024-08-17T17:41:58.779346+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:41:58.779346+0000","last_undegraded":"2024-08-17T17:41:58.779346+0000","last_fullsized":"2024-08-17T17:41:58.779346+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":290,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780316+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:41:58.780316+0000","last_peered":"2024-08-17T17:41:58.780316+0000","last_clean":"2024-08-17T17:41:58.780316+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:41:58.780316+0000","last_undegraded":"2024-08-17T17:41:58.780316+0000","last_fullsized":"2024-08-17T17:41:58.780316+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"399'9272","reported_seq":9306,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.669245+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:42:05.669245+0000","last_peered":"2024-08-17T17:42:05.669245+0000","last_clean":"2024-08-17T17:42:05.669245+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:42:05.669245+0000","last_undegraded":"2024-08-17T17:42:05.669245+0000","last_fullsized":"2024-08-17T17:42:05.669245+0000","mapping_epoch":375,"log_start":"382'2238","ondisk_log_start":"382'2238","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404488192,"num_objects":6172,"num_object_clones":0,"num_object_copies":18516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9272,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":291,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.778885+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:41:58.778885+0000","last_peered":"2024-08-17T17:41:58.778885+0000","last_clean":"2024-08-17T17:41:58.778885+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:41:58.778885+0000","last_undegraded":"2024-08-17T17:41:58.778885+0000","last_fullsized":"2024-08-17T17:41:58.778885+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":305,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780614+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:41:58.780614+0000","last_peered":"2024-08-17T17:41:58.780614+0000","last_clean":"2024-08-17T17:41:58.780614+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:41:58.780614+0000","last_undegraded":"2024-08-17T17:41:58.780614+0000","last_fullsized":"2024-08-17T17:41:58.780614+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"399'9212","reported_seq":9272,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:04.198053+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:42:04.198053+0000","last_peered":"2024-08-17T17:42:04.198053+0000","last_clean":"2024-08-17T17:42:04.198053+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:42:04.198053+0000","last_undegraded":"2024-08-17T17:42:04.198053+0000","last_fullsized":"2024-08-17T17:42:04.198053+0000","mapping_epoch":375,"log_start":"382'2200","ondisk_log_start":"382'2200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403963904,"num_objects":6164,"num_object_clones":0,"num_object_copies":18492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9212,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":415,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.781187+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:41:58.781187+0000","last_peered":"2024-08-17T17:41:58.781187+0000","last_clean":"2024-08-17T17:41:58.781187+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:41:58.781187+0000","last_undegraded":"2024-08-17T17:41:58.781187+0000","last_fullsized":"2024-08-17T17:41:58.781187+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":660,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801439+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:42:00.801439+0000","last_peered":"2024-08-17T17:42:00.801439+0000","last_clean":"2024-08-17T17:42:00.801439+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:42:00.801439+0000","last_undegraded":"2024-08-17T17:42:00.801439+0000","last_fullsized":"2024-08-17T17:42:00.801439+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":648,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811208+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:42:00.811208+0000","last_peered":"2024-08-17T17:42:00.811208+0000","last_clean":"2024-08-17T17:42:00.811208+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:42:00.811208+0000","last_undegraded":"2024-08-17T17:42:00.811208+0000","last_fullsized":"2024-08-17T17:42:00.811208+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"399'9464","reported_seq":9524,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:04.254714+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:42:04.254714+0000","last_peered":"2024-08-17T17:42:04.254714+0000","last_clean":"2024-08-17T17:42:04.254714+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:42:04.254714+0000","last_undegraded":"2024-08-17T17:42:04.254714+0000","last_fullsized":"2024-08-17T17:42:04.254714+0000","mapping_epoch":375,"log_start":"382'2400","ondisk_log_start":"382'2400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415236096,"num_objects":6336,"num_object_clones":0,"num_object_copies":19008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9464,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":738,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.805803+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:42:00.805803+0000","last_peered":"2024-08-17T17:42:00.805803+0000","last_clean":"2024-08-17T17:42:00.805803+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:42:00.805803+0000","last_undegraded":"2024-08-17T17:42:00.805803+0000","last_fullsized":"2024-08-17T17:42:00.805803+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":992,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.782332+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:41:58.782332+0000","last_peered":"2024-08-17T17:41:58.782332+0000","last_clean":"2024-08-17T17:41:58.782332+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:41:58.782332+0000","last_undegraded":"2024-08-17T17:41:58.782332+0000","last_fullsized":"2024-08-17T17:41:58.782332+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":644,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801819+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:42:00.801819+0000","last_peered":"2024-08-17T17:42:00.801819+0000","last_clean":"2024-08-17T17:42:00.801819+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:42:00.801819+0000","last_undegraded":"2024-08-17T17:42:00.801819+0000","last_fullsized":"2024-08-17T17:42:00.801819+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":304,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780224+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:41:58.780224+0000","last_peered":"2024-08-17T17:41:58.780224+0000","last_clean":"2024-08-17T17:41:58.780224+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:41:58.780224+0000","last_undegraded":"2024-08-17T17:41:58.780224+0000","last_fullsized":"2024-08-17T17:41:58.780224+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"399'9458","reported_seq":9518,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.575243+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:42:05.575243+0000","last_peered":"2024-08-17T17:42:05.575243+0000","last_clean":"2024-08-17T17:42:05.575243+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:42:05.575243+0000","last_undegraded":"2024-08-17T17:42:05.575243+0000","last_fullsized":"2024-08-17T17:42:05.575243+0000","mapping_epoch":375,"log_start":"380'300","ondisk_log_start":"380'300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9158,"log_dups_size":0,"ondisk_log_size":9158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406323200,"num_objects":6200,"num_object_clones":0,"num_object_copies":18600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9458,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":461,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.809304+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:42:00.809304+0000","last_peered":"2024-08-17T17:42:00.809304+0000","last_clean":"2024-08-17T17:42:00.809304+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:42:00.809304+0000","last_undegraded":"2024-08-17T17:42:00.809304+0000","last_fullsized":"2024-08-17T17:42:00.809304+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":778,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.782978+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:41:58.782978+0000","last_peered":"2024-08-17T17:41:58.782978+0000","last_clean":"2024-08-17T17:41:58.782978+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:41:58.782978+0000","last_undegraded":"2024-08-17T17:41:58.782978+0000","last_fullsized":"2024-08-17T17:41:58.782978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":665,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808986+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:42:00.808986+0000","last_peered":"2024-08-17T17:42:00.808986+0000","last_clean":"2024-08-17T17:42:00.808986+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:42:00.808986+0000","last_undegraded":"2024-08-17T17:42:00.808986+0000","last_fullsized":"2024-08-17T17:42:00.808986+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":662,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811524+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:42:00.811524+0000","last_peered":"2024-08-17T17:42:00.811524+0000","last_clean":"2024-08-17T17:42:00.811524+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:42:00.811524+0000","last_undegraded":"2024-08-17T17:42:00.811524+0000","last_fullsized":"2024-08-17T17:42:00.811524+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"399'9423","reported_seq":9457,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.025595+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:42:06.025595+0000","last_peered":"2024-08-17T17:42:06.025595+0000","last_clean":"2024-08-17T17:42:06.025595+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:42:06.025595+0000","last_undegraded":"2024-08-17T17:42:06.025595+0000","last_fullsized":"2024-08-17T17:42:06.025595+0000","mapping_epoch":375,"log_start":"380'500","ondisk_log_start":"380'500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8923,"log_dups_size":0,"ondisk_log_size":8923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399048704,"num_objects":6089,"num_object_clones":0,"num_object_copies":18267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9423,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":463,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.783283+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:41:58.783283+0000","last_peered":"2024-08-17T17:41:58.783283+0000","last_clean":"2024-08-17T17:41:58.783283+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:41:58.783283+0000","last_undegraded":"2024-08-17T17:41:58.783283+0000","last_fullsized":"2024-08-17T17:41:58.783283+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":467,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801552+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:42:00.801552+0000","last_peered":"2024-08-17T17:42:00.801552+0000","last_clean":"2024-08-17T17:42:00.801552+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:42:00.801552+0000","last_undegraded":"2024-08-17T17:42:00.801552+0000","last_fullsized":"2024-08-17T17:42:00.801552+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":655,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808711+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:42:00.808711+0000","last_peered":"2024-08-17T17:42:00.808711+0000","last_clean":"2024-08-17T17:42:00.808711+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:42:00.808711+0000","last_undegraded":"2024-08-17T17:42:00.808711+0000","last_fullsized":"2024-08-17T17:42:00.808711+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":702,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806644+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:42:00.806644+0000","last_peered":"2024-08-17T17:42:00.806644+0000","last_clean":"2024-08-17T17:42:00.806644+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:42:00.806644+0000","last_undegraded":"2024-08-17T17:42:00.806644+0000","last_fullsized":"2024-08-17T17:42:00.806644+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"399'9419","reported_seq":9479,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.584766+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:42:05.584766+0000","last_peered":"2024-08-17T17:42:05.584766+0000","last_clean":"2024-08-17T17:42:05.584766+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:42:05.584766+0000","last_undegraded":"2024-08-17T17:42:05.584766+0000","last_fullsized":"2024-08-17T17:42:05.584766+0000","mapping_epoch":375,"log_start":"380'300","ondisk_log_start":"380'300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9119,"log_dups_size":0,"ondisk_log_size":9119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402980864,"num_objects":6149,"num_object_clones":0,"num_object_copies":18447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9419,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":415,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.782478+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:41:58.782478+0000","last_peered":"2024-08-17T17:41:58.782478+0000","last_clean":"2024-08-17T17:41:58.782478+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:41:58.782478+0000","last_undegraded":"2024-08-17T17:41:58.782478+0000","last_fullsized":"2024-08-17T17:41:58.782478+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1115,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.781348+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:41:58.781348+0000","last_peered":"2024-08-17T17:41:58.781348+0000","last_clean":"2024-08-17T17:41:58.781348+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:41:58.781348+0000","last_undegraded":"2024-08-17T17:41:58.781348+0000","last_fullsized":"2024-08-17T17:41:58.781348+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":480,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.810972+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:42:00.810972+0000","last_peered":"2024-08-17T17:42:00.810972+0000","last_clean":"2024-08-17T17:42:00.810972+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:42:00.810972+0000","last_undegraded":"2024-08-17T17:42:00.810972+0000","last_fullsized":"2024-08-17T17:42:00.810972+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":46,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779628+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:41:58.779628+0000","last_peered":"2024-08-17T17:41:58.779628+0000","last_clean":"2024-08-17T17:41:58.779628+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:41:58.779628+0000","last_undegraded":"2024-08-17T17:41:58.779628+0000","last_fullsized":"2024-08-17T17:41:58.779628+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"399'9344","reported_seq":9377,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.030553+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:42:06.030553+0000","last_peered":"2024-08-17T17:42:06.030553+0000","last_clean":"2024-08-17T17:42:06.030553+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:42:06.030553+0000","last_undegraded":"2024-08-17T17:42:06.030553+0000","last_fullsized":"2024-08-17T17:42:06.030553+0000","mapping_epoch":375,"log_start":"380'500","ondisk_log_start":"380'500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403963904,"num_objects":6164,"num_object_clones":0,"num_object_copies":18492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9344,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":458,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807892+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:42:00.807892+0000","last_peered":"2024-08-17T17:42:00.807892+0000","last_clean":"2024-08-17T17:42:00.807892+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:42:00.807892+0000","last_undegraded":"2024-08-17T17:42:00.807892+0000","last_fullsized":"2024-08-17T17:42:00.807892+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780226+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:41:58.780226+0000","last_peered":"2024-08-17T17:41:58.780226+0000","last_clean":"2024-08-17T17:41:58.780226+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:41:58.780226+0000","last_undegraded":"2024-08-17T17:41:58.780226+0000","last_fullsized":"2024-08-17T17:41:58.780226+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":947,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806791+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:42:00.806791+0000","last_peered":"2024-08-17T17:42:00.806791+0000","last_clean":"2024-08-17T17:42:00.806791+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:42:00.806791+0000","last_undegraded":"2024-08-17T17:42:00.806791+0000","last_fullsized":"2024-08-17T17:42:00.806791+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":671,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807078+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:42:00.807078+0000","last_peered":"2024-08-17T17:42:00.807078+0000","last_clean":"2024-08-17T17:42:00.807078+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:42:00.807078+0000","last_undegraded":"2024-08-17T17:42:00.807078+0000","last_fullsized":"2024-08-17T17:42:00.807078+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"399'9549","reported_seq":9583,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.673701+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:42:05.673701+0000","last_peered":"2024-08-17T17:42:05.673701+0000","last_clean":"2024-08-17T17:42:05.673701+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:42:05.673701+0000","last_undegraded":"2024-08-17T17:42:05.673701+0000","last_fullsized":"2024-08-17T17:42:05.673701+0000","mapping_epoch":375,"log_start":"383'2503","ondisk_log_start":"383'2503","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7046,"log_dups_size":0,"ondisk_log_size":7046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408092672,"num_objects":6227,"num_object_clones":0,"num_object_copies":18681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9549,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":464,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811238+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:42:00.811238+0000","last_peered":"2024-08-17T17:42:00.811238+0000","last_clean":"2024-08-17T17:42:00.811238+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:42:00.811238+0000","last_undegraded":"2024-08-17T17:42:00.811238+0000","last_fullsized":"2024-08-17T17:42:00.811238+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":668,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806704+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:42:00.806704+0000","last_peered":"2024-08-17T17:42:00.806704+0000","last_clean":"2024-08-17T17:42:00.806704+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:42:00.806704+0000","last_undegraded":"2024-08-17T17:42:00.806704+0000","last_fullsized":"2024-08-17T17:42:00.806704+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":967,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.798142+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:41:58.798142+0000","last_peered":"2024-08-17T17:41:58.798142+0000","last_clean":"2024-08-17T17:41:58.798142+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:41:58.798142+0000","last_undegraded":"2024-08-17T17:41:58.798142+0000","last_fullsized":"2024-08-17T17:41:58.798142+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":665,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.809795+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:42:00.809795+0000","last_peered":"2024-08-17T17:42:00.809795+0000","last_clean":"2024-08-17T17:42:00.809795+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:42:00.809795+0000","last_undegraded":"2024-08-17T17:42:00.809795+0000","last_fullsized":"2024-08-17T17:42:00.809795+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"399'9542","reported_seq":9575,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.640857+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:42:05.640857+0000","last_peered":"2024-08-17T17:42:05.640857+0000","last_clean":"2024-08-17T17:42:05.640857+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:42:05.640857+0000","last_undegraded":"2024-08-17T17:42:05.640857+0000","last_fullsized":"2024-08-17T17:42:05.640857+0000","mapping_epoch":375,"log_start":"383'2514","ondisk_log_start":"383'2514","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409665560,"num_objects":6252,"num_object_clones":0,"num_object_copies":18756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6252,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9542,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":415,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780098+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:41:58.780098+0000","last_peered":"2024-08-17T17:41:58.780098+0000","last_clean":"2024-08-17T17:41:58.780098+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:41:58.780098+0000","last_undegraded":"2024-08-17T17:41:58.780098+0000","last_fullsized":"2024-08-17T17:41:58.780098+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":658,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808184+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:42:00.808184+0000","last_peered":"2024-08-17T17:42:00.808184+0000","last_clean":"2024-08-17T17:42:00.808184+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:42:00.808184+0000","last_undegraded":"2024-08-17T17:42:00.808184+0000","last_fullsized":"2024-08-17T17:42:00.808184+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1178,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807915+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:42:00.807915+0000","last_peered":"2024-08-17T17:42:00.807915+0000","last_clean":"2024-08-17T17:42:00.807915+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:42:00.807915+0000","last_undegraded":"2024-08-17T17:42:00.807915+0000","last_fullsized":"2024-08-17T17:42:00.807915+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":656,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801180+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:42:00.801180+0000","last_peered":"2024-08-17T17:42:00.801180+0000","last_clean":"2024-08-17T17:42:00.801180+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:42:00.801180+0000","last_undegraded":"2024-08-17T17:42:00.801180+0000","last_fullsized":"2024-08-17T17:42:00.801180+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"399'9515","reported_seq":9575,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:04.274770+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:42:04.274770+0000","last_peered":"2024-08-17T17:42:04.274770+0000","last_clean":"2024-08-17T17:42:04.274770+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:42:04.274770+0000","last_undegraded":"2024-08-17T17:42:04.274770+0000","last_fullsized":"2024-08-17T17:42:04.274770+0000","mapping_epoch":375,"log_start":"383'2500","ondisk_log_start":"383'2500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7015,"log_dups_size":0,"ondisk_log_size":7015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422117376,"num_objects":6441,"num_object_clones":0,"num_object_copies":19323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9515,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":458,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.802481+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:42:00.802481+0000","last_peered":"2024-08-17T17:42:00.802481+0000","last_clean":"2024-08-17T17:42:00.802481+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:42:00.802481+0000","last_undegraded":"2024-08-17T17:42:00.802481+0000","last_fullsized":"2024-08-17T17:42:00.802481+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":677,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811339+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:42:00.811339+0000","last_peered":"2024-08-17T17:42:00.811339+0000","last_clean":"2024-08-17T17:42:00.811339+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:42:00.811339+0000","last_undegraded":"2024-08-17T17:42:00.811339+0000","last_fullsized":"2024-08-17T17:42:00.811339+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":31,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.783363+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:41:58.783363+0000","last_peered":"2024-08-17T17:41:58.783363+0000","last_clean":"2024-08-17T17:41:58.783363+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:41:58.783363+0000","last_undegraded":"2024-08-17T17:41:58.783363+0000","last_fullsized":"2024-08-17T17:41:58.783363+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":342,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779773+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:41:58.779773+0000","last_peered":"2024-08-17T17:41:58.779773+0000","last_clean":"2024-08-17T17:41:58.779773+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:41:58.779773+0000","last_undegraded":"2024-08-17T17:41:58.779773+0000","last_fullsized":"2024-08-17T17:41:58.779773+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":305,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779928+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:41:58.779928+0000","last_peered":"2024-08-17T17:41:58.779928+0000","last_clean":"2024-08-17T17:41:58.779928+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:41:58.779928+0000","last_undegraded":"2024-08-17T17:41:58.779928+0000","last_fullsized":"2024-08-17T17:41:58.779928+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":315,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.782020+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:41:58.782020+0000","last_peered":"2024-08-17T17:41:58.782020+0000","last_clean":"2024-08-17T17:41:58.782020+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:41:58.782020+0000","last_undegraded":"2024-08-17T17:41:58.782020+0000","last_fullsized":"2024-08-17T17:41:58.782020+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":653,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806924+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:42:00.806924+0000","last_peered":"2024-08-17T17:42:00.806924+0000","last_clean":"2024-08-17T17:42:00.806924+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:42:00.806924+0000","last_undegraded":"2024-08-17T17:42:00.806924+0000","last_fullsized":"2024-08-17T17:42:00.806924+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":400,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.798060+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:41:58.798060+0000","last_peered":"2024-08-17T17:41:58.798060+0000","last_clean":"2024-08-17T17:41:58.798060+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:41:58.798060+0000","last_undegraded":"2024-08-17T17:41:58.798060+0000","last_fullsized":"2024-08-17T17:41:58.798060+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":183,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779977+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:41:58.779977+0000","last_peered":"2024-08-17T17:41:58.779977+0000","last_clean":"2024-08-17T17:41:58.779977+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:41:58.779977+0000","last_undegraded":"2024-08-17T17:41:58.779977+0000","last_fullsized":"2024-08-17T17:41:58.779977+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":620,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801316+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:42:00.801316+0000","last_peered":"2024-08-17T17:42:00.801316+0000","last_clean":"2024-08-17T17:42:00.801316+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:42:00.801316+0000","last_undegraded":"2024-08-17T17:42:00.801316+0000","last_fullsized":"2024-08-17T17:42:00.801316+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":679,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811657+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:42:00.811657+0000","last_peered":"2024-08-17T17:42:00.811657+0000","last_clean":"2024-08-17T17:42:00.811657+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:42:00.811657+0000","last_undegraded":"2024-08-17T17:42:00.811657+0000","last_fullsized":"2024-08-17T17:42:00.811657+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":669,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801075+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:42:00.801075+0000","last_peered":"2024-08-17T17:42:00.801075+0000","last_clean":"2024-08-17T17:42:00.801075+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:42:00.801075+0000","last_undegraded":"2024-08-17T17:42:00.801075+0000","last_fullsized":"2024-08-17T17:42:00.801075+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":194,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780878+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:41:58.780878+0000","last_peered":"2024-08-17T17:41:58.780878+0000","last_clean":"2024-08-17T17:41:58.780878+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:41:58.780878+0000","last_undegraded":"2024-08-17T17:41:58.780878+0000","last_fullsized":"2024-08-17T17:41:58.780878+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":674,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801346+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:42:00.801346+0000","last_peered":"2024-08-17T17:42:00.801346+0000","last_clean":"2024-08-17T17:42:00.801346+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:42:00.801346+0000","last_undegraded":"2024-08-17T17:42:00.801346+0000","last_fullsized":"2024-08-17T17:42:00.801346+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":652,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811472+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:42:00.811472+0000","last_peered":"2024-08-17T17:42:00.811472+0000","last_clean":"2024-08-17T17:42:00.811472+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:42:00.811472+0000","last_undegraded":"2024-08-17T17:42:00.811472+0000","last_fullsized":"2024-08-17T17:42:00.811472+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":656,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806362+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:42:00.806362+0000","last_peered":"2024-08-17T17:42:00.806362+0000","last_clean":"2024-08-17T17:42:00.806362+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:42:00.806362+0000","last_undegraded":"2024-08-17T17:42:00.806362+0000","last_fullsized":"2024-08-17T17:42:00.806362+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":671,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801867+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:42:00.801867+0000","last_peered":"2024-08-17T17:42:00.801867+0000","last_clean":"2024-08-17T17:42:00.801867+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:42:00.801867+0000","last_undegraded":"2024-08-17T17:42:00.801867+0000","last_fullsized":"2024-08-17T17:42:00.801867+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":289,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779495+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:41:58.779495+0000","last_peered":"2024-08-17T17:41:58.779495+0000","last_clean":"2024-08-17T17:41:58.779495+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:41:58.779495+0000","last_undegraded":"2024-08-17T17:41:58.779495+0000","last_fullsized":"2024-08-17T17:41:58.779495+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":184,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807254+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:42:00.807254+0000","last_peered":"2024-08-17T17:42:00.807254+0000","last_clean":"2024-08-17T17:42:00.807254+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:42:00.807254+0000","last_undegraded":"2024-08-17T17:42:00.807254+0000","last_fullsized":"2024-08-17T17:42:00.807254+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":46,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.809000+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:42:00.809000+0000","last_peered":"2024-08-17T17:42:00.809000+0000","last_clean":"2024-08-17T17:42:00.809000+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:42:00.809000+0000","last_undegraded":"2024-08-17T17:42:00.809000+0000","last_fullsized":"2024-08-17T17:42:00.809000+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":289,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779857+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:41:58.779857+0000","last_peered":"2024-08-17T17:41:58.779857+0000","last_clean":"2024-08-17T17:41:58.779857+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:41:58.779857+0000","last_undegraded":"2024-08-17T17:41:58.779857+0000","last_fullsized":"2024-08-17T17:41:58.779857+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":187,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779889+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:41:58.779889+0000","last_peered":"2024-08-17T17:41:58.779889+0000","last_clean":"2024-08-17T17:41:58.779889+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:41:58.779889+0000","last_undegraded":"2024-08-17T17:41:58.779889+0000","last_fullsized":"2024-08-17T17:41:58.779889+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":289,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.779415+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:41:58.779415+0000","last_peered":"2024-08-17T17:41:58.779415+0000","last_clean":"2024-08-17T17:41:58.779415+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:41:58.779415+0000","last_undegraded":"2024-08-17T17:41:58.779415+0000","last_fullsized":"2024-08-17T17:41:58.779415+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":197,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.780345+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:41:58.780345+0000","last_peered":"2024-08-17T17:41:58.780345+0000","last_clean":"2024-08-17T17:41:58.780345+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:41:58.780345+0000","last_undegraded":"2024-08-17T17:41:58.780345+0000","last_fullsized":"2024-08-17T17:41:58.780345+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":676,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.801635+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:42:00.801635+0000","last_peered":"2024-08-17T17:42:00.801635+0000","last_clean":"2024-08-17T17:42:00.801635+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:42:00.801635+0000","last_undegraded":"2024-08-17T17:42:00.801635+0000","last_fullsized":"2024-08-17T17:42:00.801635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":677,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.802630+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:42:00.802630+0000","last_peered":"2024-08-17T17:42:00.802630+0000","last_clean":"2024-08-17T17:42:00.802630+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:42:00.802630+0000","last_undegraded":"2024-08-17T17:42:00.802630+0000","last_fullsized":"2024-08-17T17:42:00.802630+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":305,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:41:58.781435+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:41:58.781435+0000","last_peered":"2024-08-17T17:41:58.781435+0000","last_clean":"2024-08-17T17:41:58.781435+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:41:58.781435+0000","last_undegraded":"2024-08-17T17:41:58.781435+0000","last_fullsized":"2024-08-17T17:41:58.781435+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":696,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811573+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:42:00.811573+0000","last_peered":"2024-08-17T17:42:00.811573+0000","last_clean":"2024-08-17T17:42:00.811573+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:42:00.811573+0000","last_undegraded":"2024-08-17T17:42:00.811573+0000","last_fullsized":"2024-08-17T17:42:00.811573+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6512115736,"num_objects":99368,"num_object_clones":0,"num_object_copies":298104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1185,"num_objects_misplaced":6218,"num_objects_unfound":0,"num_objects_dirty":99368,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150618,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19615264768,"data_stored":19615252552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126463,"ondisk_log_size":126463,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201743,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3983828,"kb_used_data":3200460,"kb_used_omap":1,"kb_used_meta":783358,"kb_avail":89761324,"statfs":{"total":95995035648,"available":91915595776,"internally_reserved":0,"allocated":3277271040,"data_stored":3275300864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":802159051},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922239,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1549824,"kb_used_data":1209528,"kb_used_omap":7,"kb_used_meta":340280,"kb_avail":92195328,"statfs":{"total":95995035648,"available":94408015872,"internally_reserved":0,"allocated":1238556672,"data_stored":1236605918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":348447414},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085763,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4040584,"kb_used_data":3588480,"kb_used_omap":7,"kb_used_meta":452088,"kb_avail":89704568,"statfs":{"total":95995035648,"available":91857477632,"internally_reserved":0,"allocated":3674603520,"data_stored":3672641875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":462938553},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986653,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1910028,"kb_used_data":1215236,"kb_used_omap":1,"kb_used_meta":694782,"kb_avail":91835124,"statfs":{"total":95995035648,"available":94039166976,"internally_reserved":0,"allocated":1244401664,"data_stored":1242439594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":711457222},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435544,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4132028,"kb_used_data":3234036,"kb_used_omap":5,"kb_used_meta":897978,"kb_avail":89613124,"statfs":{"total":95995035648,"available":91763838976,"internally_reserved":0,"allocated":3311652864,"data_stored":3309702922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":919529545},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599071,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1981180,"kb_used_data":1173428,"kb_used_omap":7,"kb_used_meta":807736,"kb_avail":91763972,"statfs":{"total":95995035648,"available":93966307328,"internally_reserved":0,"allocated":1201590272,"data_stored":1199639679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":827121837},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762597,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4816500,"kb_used_data":4048876,"kb_used_omap":7,"kb_used_meta":767480,"kb_avail":88928652,"statfs":{"total":95995035648,"available":91062939648,"internally_reserved":0,"allocated":4146049024,"data_stored":4144105233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":785899937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926123,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407744,"kb_used_data":1609720,"kb_used_omap":7,"kb_used_meta":798008,"kb_avail":91337408,"statfs":{"total":95995035648,"available":93529505792,"internally_reserved":0,"allocated":1648353280,"data_stored":1646399485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":817160796},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1627979776,"data_stored":1627979776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4127657984,"data_stored":4127653912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1194000384,"data_stored":1194000384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3300261888,"data_stored":3300261888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1231028224,"data_stored":1231028224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3644788736,"data_stored":3644784664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1225981952,"data_stored":1225981952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3263565824,"data_stored":3263561752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:11.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:11.078 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:11.082 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:11.111 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:11.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[109246]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 400 KiB/s, 6 objects/s recovering 2024-08-17T17:42:11.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:11.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:11.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[104516]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 400 KiB/s, 6 objects/s recovering 2024-08-17T17:42:11.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:11.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:11.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:10 smithi175 ceph-mon[101434]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 1185/298188 objects degraded (0.397%); 6218/298188 objects misplaced (2.085%); 400 KiB/s, 6 objects/s recovering 2024-08-17T17:42:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:11.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.259 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:11.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:11.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:11.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:11.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:11.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:11.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:11.661 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:11.701 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:11.701 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:11.763 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:11.794 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:11.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:11.924 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:11.928 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:12.030 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:12.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:12.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:11 smithi064 ceph-mon[104516]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:12.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:11 smithi064 ceph-mon[109246]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:12.201 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:12.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:12.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:12.358 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:11 smithi175 ceph-mon[101434]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:12.396 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:42:12.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:12.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:12.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:12.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:42:12.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:12.726 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:12.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:12.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:12.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:13.019 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:13.052 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:13.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:13.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:13.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:13.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:13.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:13.455 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:13.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.597 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:13.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:13.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:13.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:13.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:13.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:13.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:14.017 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:14.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:14.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:14.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.306 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:14.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:14.316 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:42:13 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=infra.usagestats t=2024-08-17T17:42:13.918386273Z level=info msg="Usage stats are ready to report" 2024-08-17T17:42:14.338 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:13] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:42:14.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:14.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:14.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:14.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:14.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:14 smithi175 ceph-mon[101434]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:14 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1351/285738 objects degraded (0.473%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:14.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:14 smithi064 ceph-mon[104516]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:14 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1351/285738 objects degraded (0.473%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:14 smithi064 ceph-mon[109246]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 1351/285738 objects degraded (0.473%); 5989/285738 objects misplaced (2.096%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:14 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1351/285738 objects degraded (0.473%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:14.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:14.815 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:14.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:14.940 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:14.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:15.015 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:15.015 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:15.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:15.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:15.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:15.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:15.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:15.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:15.480 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:15.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:15.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:15.646 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:15.725 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:42:15.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:15.793 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:15.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:15.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:16.030 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:16.071 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:16.073 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:16.175 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:16.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:16.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:16.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:16.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:16.480 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:16 smithi175 ceph-mon[101434]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1351/282585 objects degraded (0.478%); 5989/282585 objects misplaced (2.119%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:16.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:16.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:16 smithi064 ceph-mon[104516]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1351/282585 objects degraded (0.478%); 5989/282585 objects misplaced (2.119%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:16.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:16 smithi064 ceph-mon[109246]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1351/282585 objects degraded (0.478%); 5989/282585 objects misplaced (2.119%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:42:16.619 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:16.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:16.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:16.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:16.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:16.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:17.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:17.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:17.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:17.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:17.310 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:17.345 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:17.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:17.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:17.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:17.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:17.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:17.627 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:42:17.628 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:17.660 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:42:17.660 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:17.689 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:42:17.690 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:17.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:17.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:42:17.724 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:17.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:42:17.756 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:17.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:17.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:17.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:42:17.792 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:17.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:42:17.829 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:17.874 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:17.876 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:42:17.876 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:17.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:17.944 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:42:17.944 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:42:17.944 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:42:17.944 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:42:17.944 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:42:18.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:18.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:18.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:18.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:18.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:18.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:18.427 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:18.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:18.508 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:42:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:42:18.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:18.590 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:18.635 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:18 smithi175 ceph-mon[101434]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:42:18.636 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1262432076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:18.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:42:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:18 smithi064 ceph-mon[104516]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:42:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1262432076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:18 smithi064 ceph-mon[109246]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:42:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1262432076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:18.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:18.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:18.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:18.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:18.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:18.996 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:19.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:19.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:19.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:19.181 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:19.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:19.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:19.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:19.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:19 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1556/273438 objects degraded (0.569%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:19.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:19 smithi175 ceph-mon[101434]: from='client.35910 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:19.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:19.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:19.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:19.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:19 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1556/273438 objects degraded (0.569%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:19.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:19 smithi064 ceph-mon[104516]: from='client.35910 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:19.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:19 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1556/273438 objects degraded (0.569%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:19.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:19 smithi064 ceph-mon[109246]: from='client.35910 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:19.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:19.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:19.708 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1239,"stamp":"2024-08-17T17:42:17.277523+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5976233981,"num_objects":91146,"num_object_clones":0,"num_object_copies":273438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1556,"num_objects_misplaced":5725,"num_objects_unfound":0,"num_objects_dirty":91146,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":159776,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":639,"num_bytes_recovered":42342627,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129085,"ondisk_log_size":129085,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23264628,"kb_used_data":17735092,"kb_used_omap":46,"kb_used_meta":5529361,"kb_avail":726696588,"statfs":{"total":767960285184,"available":744137306112,"internally_reserved":0,"allocated":18160734208,"data_stored":18145060146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5662065907},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-540672000,"num_objects":-8250,"num_object_clones":0,"num_object_copies":-24750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":371,"num_objects_misplaced":-493,"num_objects_unfound":0,"num_objects_dirty":-8250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8250,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":7995392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003857"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":660,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883515+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:42:07.883515+0000","last_peered":"2024-08-17T17:42:07.883515+0000","last_clean":"2024-08-17T17:42:07.883515+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:42:07.883515+0000","last_undegraded":"2024-08-17T17:42:07.883515+0000","last_fullsized":"2024-08-17T17:42:07.883515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":400,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808146+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:42:00.808146+0000","last_peered":"2024-08-17T17:42:00.808146+0000","last_clean":"2024-08-17T17:42:00.808146+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:42:00.808146+0000","last_undegraded":"2024-08-17T17:42:00.808146+0000","last_fullsized":"2024-08-17T17:42:00.808146+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":184,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803908+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:42:00.803908+0000","last_peered":"2024-08-17T17:42:00.803908+0000","last_clean":"2024-08-17T17:42:00.803908+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:42:00.803908+0000","last_undegraded":"2024-08-17T17:42:00.803908+0000","last_fullsized":"2024-08-17T17:42:00.803908+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":652,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882013+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:42:07.882013+0000","last_peered":"2024-08-17T17:42:07.882013+0000","last_clean":"2024-08-17T17:42:07.882013+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:42:07.882013+0000","last_undegraded":"2024-08-17T17:42:07.882013+0000","last_fullsized":"2024-08-17T17:42:07.882013+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":652,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882576+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:42:07.882576+0000","last_peered":"2024-08-17T17:42:07.882576+0000","last_clean":"2024-08-17T17:42:07.882576+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:42:07.882576+0000","last_undegraded":"2024-08-17T17:42:07.882576+0000","last_fullsized":"2024-08-17T17:42:07.882576+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806284+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:42:00.806284+0000","last_peered":"2024-08-17T17:42:00.806284+0000","last_clean":"2024-08-17T17:42:00.806284+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:42:00.806284+0000","last_undegraded":"2024-08-17T17:42:00.806284+0000","last_fullsized":"2024-08-17T17:42:00.806284+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":660,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.888591+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:42:07.888591+0000","last_peered":"2024-08-17T17:42:07.888591+0000","last_clean":"2024-08-17T17:42:07.888591+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:42:07.888591+0000","last_undegraded":"2024-08-17T17:42:07.888591+0000","last_fullsized":"2024-08-17T17:42:07.888591+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":334,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.814746+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:42:00.814746+0000","last_peered":"2024-08-17T17:42:00.814746+0000","last_clean":"2024-08-17T17:42:00.814746+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:42:00.814746+0000","last_undegraded":"2024-08-17T17:42:00.814746+0000","last_fullsized":"2024-08-17T17:42:00.814746+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"400'9896","reported_seq":10994,"reported_epoch":400,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:15.660786+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:42:15.660786+0000","last_peered":"2024-08-17T17:42:15.660786+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:42:15.660786+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"380'800","ondisk_log_start":"380'800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9096,"log_dups_size":0,"ondisk_log_size":9096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376700928,"num_objects":5748,"num_object_clones":0,"num_object_copies":17244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1556,"num_objects_misplaced":5725,"num_objects_unfound":0,"num_objects_dirty":5748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9896,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":518,"num_bytes_recovered":33947648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":5725},{"shards":"0,5","objects":-1533}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":396,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808625+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:42:00.808625+0000","last_peered":"2024-08-17T17:42:00.808625+0000","last_clean":"2024-08-17T17:42:00.808625+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:42:00.808625+0000","last_undegraded":"2024-08-17T17:42:00.808625+0000","last_fullsized":"2024-08-17T17:42:00.808625+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.043993+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:42:06.043993+0000","last_peered":"2024-08-17T17:42:06.043993+0000","last_clean":"2024-08-17T17:42:06.043993+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:42:06.043993+0000","last_undegraded":"2024-08-17T17:42:06.043993+0000","last_fullsized":"2024-08-17T17:42:06.043993+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"400'9960","reported_seq":10022,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.268752+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:42:16.268752+0000","last_peered":"2024-08-17T17:42:16.268752+0000","last_clean":"2024-08-17T17:42:16.268752+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:42:16.268752+0000","last_undegraded":"2024-08-17T17:42:16.268752+0000","last_fullsized":"2024-08-17T17:42:16.268752+0000","mapping_epoch":375,"log_start":"383'2800","ondisk_log_start":"383'2800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372244480,"num_objects":5680,"num_object_clones":0,"num_object_copies":17040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9960,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":397,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806131+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:42:00.806131+0000","last_peered":"2024-08-17T17:42:00.806131+0000","last_clean":"2024-08-17T17:42:00.806131+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:42:00.806131+0000","last_undegraded":"2024-08-17T17:42:00.806131+0000","last_fullsized":"2024-08-17T17:42:00.806131+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":292,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806511+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:42:00.806511+0000","last_peered":"2024-08-17T17:42:00.806511+0000","last_clean":"2024-08-17T17:42:00.806511+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:42:00.806511+0000","last_undegraded":"2024-08-17T17:42:00.806511+0000","last_fullsized":"2024-08-17T17:42:00.806511+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"400'9961","reported_seq":9996,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.015696+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:42:16.015696+0000","last_peered":"2024-08-17T17:42:16.015696+0000","last_clean":"2024-08-17T17:42:16.015696+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:42:16.015696+0000","last_undegraded":"2024-08-17T17:42:16.015696+0000","last_fullsized":"2024-08-17T17:42:16.015696+0000","mapping_epoch":375,"log_start":"382'1100","ondisk_log_start":"382'1100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371916800,"num_objects":5675,"num_object_clones":0,"num_object_copies":17025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9961,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":414,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.810925+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:42:00.810925+0000","last_peered":"2024-08-17T17:42:00.810925+0000","last_clean":"2024-08-17T17:42:00.810925+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:42:00.810925+0000","last_undegraded":"2024-08-17T17:42:00.810925+0000","last_fullsized":"2024-08-17T17:42:00.810925+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":314,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.811167+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:42:00.811167+0000","last_peered":"2024-08-17T17:42:00.811167+0000","last_clean":"2024-08-17T17:42:00.811167+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:42:00.811167+0000","last_undegraded":"2024-08-17T17:42:00.811167+0000","last_fullsized":"2024-08-17T17:42:00.811167+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"400'10002","reported_seq":10037,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.014219+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:42:16.014219+0000","last_peered":"2024-08-17T17:42:16.014219+0000","last_clean":"2024-08-17T17:42:16.014219+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:42:16.014219+0000","last_undegraded":"2024-08-17T17:42:16.014219+0000","last_fullsized":"2024-08-17T17:42:16.014219+0000","mapping_epoch":375,"log_start":"382'1100","ondisk_log_start":"382'1100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8902,"log_dups_size":0,"ondisk_log_size":8902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371458048,"num_objects":5668,"num_object_clones":0,"num_object_copies":17004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.804359+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:42:00.804359+0000","last_peered":"2024-08-17T17:42:00.804359+0000","last_clean":"2024-08-17T17:42:00.804359+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:42:00.804359+0000","last_undegraded":"2024-08-17T17:42:00.804359+0000","last_fullsized":"2024-08-17T17:42:00.804359+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":667,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882756+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:42:07.882756+0000","last_peered":"2024-08-17T17:42:07.882756+0000","last_clean":"2024-08-17T17:42:07.882756+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:42:07.882756+0000","last_undegraded":"2024-08-17T17:42:07.882756+0000","last_fullsized":"2024-08-17T17:42:07.882756+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"400'9921","reported_seq":9983,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:14.237820+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:42:14.237820+0000","last_peered":"2024-08-17T17:42:14.237820+0000","last_clean":"2024-08-17T17:42:14.237820+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:42:14.237820+0000","last_undegraded":"2024-08-17T17:42:14.237820+0000","last_fullsized":"2024-08-17T17:42:14.237820+0000","mapping_epoch":375,"log_start":"383'2900","ondisk_log_start":"383'2900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383057920,"num_objects":5845,"num_object_clones":0,"num_object_copies":17535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9921,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":755,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806203+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:42:00.806203+0000","last_peered":"2024-08-17T17:42:00.806203+0000","last_clean":"2024-08-17T17:42:00.806203+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:42:00.806203+0000","last_undegraded":"2024-08-17T17:42:00.806203+0000","last_fullsized":"2024-08-17T17:42:00.806203+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":682,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884122+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:42:07.884122+0000","last_peered":"2024-08-17T17:42:07.884122+0000","last_clean":"2024-08-17T17:42:07.884122+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:42:07.884122+0000","last_undegraded":"2024-08-17T17:42:07.884122+0000","last_fullsized":"2024-08-17T17:42:07.884122+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":315,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808301+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:42:00.808301+0000","last_peered":"2024-08-17T17:42:00.808301+0000","last_clean":"2024-08-17T17:42:00.808301+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:42:00.808301+0000","last_undegraded":"2024-08-17T17:42:00.808301+0000","last_fullsized":"2024-08-17T17:42:00.808301+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"400'9726","reported_seq":9788,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.255247+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:42:16.255247+0000","last_peered":"2024-08-17T17:42:16.255247+0000","last_clean":"2024-08-17T17:42:16.255247+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:42:16.255247+0000","last_undegraded":"2024-08-17T17:42:16.255247+0000","last_fullsized":"2024-08-17T17:42:16.255247+0000","mapping_epoch":375,"log_start":"383'2500","ondisk_log_start":"383'2500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7226,"log_dups_size":0,"ondisk_log_size":7226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361234432,"num_objects":5512,"num_object_clones":0,"num_object_copies":16536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9726,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":416,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803059+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:42:00.803059+0000","last_peered":"2024-08-17T17:42:00.803059+0000","last_clean":"2024-08-17T17:42:00.803059+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:42:00.803059+0000","last_undegraded":"2024-08-17T17:42:00.803059+0000","last_fullsized":"2024-08-17T17:42:00.803059+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":321,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806839+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:42:00.806839+0000","last_peered":"2024-08-17T17:42:00.806839+0000","last_clean":"2024-08-17T17:42:00.806839+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:42:00.806839+0000","last_undegraded":"2024-08-17T17:42:00.806839+0000","last_fullsized":"2024-08-17T17:42:00.806839+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":291,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807670+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:42:00.807670+0000","last_peered":"2024-08-17T17:42:00.807670+0000","last_clean":"2024-08-17T17:42:00.807670+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:42:00.807670+0000","last_undegraded":"2024-08-17T17:42:00.807670+0000","last_fullsized":"2024-08-17T17:42:00.807670+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"400'9806","reported_seq":9841,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.732694+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:42:15.732694+0000","last_peered":"2024-08-17T17:42:15.732694+0000","last_clean":"2024-08-17T17:42:15.732694+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:42:15.732694+0000","last_undegraded":"2024-08-17T17:42:15.732694+0000","last_fullsized":"2024-08-17T17:42:15.732694+0000","mapping_epoch":375,"log_start":"383'2738","ondisk_log_start":"383'2738","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369491968,"num_objects":5638,"num_object_clones":0,"num_object_copies":16914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9806,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":292,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806384+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:42:00.806384+0000","last_peered":"2024-08-17T17:42:00.806384+0000","last_clean":"2024-08-17T17:42:00.806384+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:42:00.806384+0000","last_undegraded":"2024-08-17T17:42:00.806384+0000","last_fullsized":"2024-08-17T17:42:00.806384+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808404+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:42:00.808404+0000","last_peered":"2024-08-17T17:42:00.808404+0000","last_clean":"2024-08-17T17:42:00.808404+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:42:00.808404+0000","last_undegraded":"2024-08-17T17:42:00.808404+0000","last_fullsized":"2024-08-17T17:42:00.808404+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"400'9718","reported_seq":9780,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:14.153075+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:42:14.153075+0000","last_peered":"2024-08-17T17:42:14.153075+0000","last_clean":"2024-08-17T17:42:14.153075+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:42:14.153075+0000","last_undegraded":"2024-08-17T17:42:14.153075+0000","last_fullsized":"2024-08-17T17:42:14.153075+0000","mapping_epoch":375,"log_start":"383'2700","ondisk_log_start":"383'2700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370802688,"num_objects":5658,"num_object_clones":0,"num_object_copies":16974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9718,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":416,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806580+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:42:00.806580+0000","last_peered":"2024-08-17T17:42:00.806580+0000","last_clean":"2024-08-17T17:42:00.806580+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:42:00.806580+0000","last_undegraded":"2024-08-17T17:42:00.806580+0000","last_fullsized":"2024-08-17T17:42:00.806580+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":662,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884401+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:42:07.884401+0000","last_peered":"2024-08-17T17:42:07.884401+0000","last_clean":"2024-08-17T17:42:07.884401+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:42:07.884401+0000","last_undegraded":"2024-08-17T17:42:07.884401+0000","last_fullsized":"2024-08-17T17:42:07.884401+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":650,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.890303+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:42:07.890303+0000","last_peered":"2024-08-17T17:42:07.890303+0000","last_clean":"2024-08-17T17:42:07.890303+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:42:07.890303+0000","last_undegraded":"2024-08-17T17:42:07.890303+0000","last_fullsized":"2024-08-17T17:42:07.890303+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"400'10029","reported_seq":10091,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:14.185280+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:42:14.185280+0000","last_peered":"2024-08-17T17:42:14.185280+0000","last_clean":"2024-08-17T17:42:14.185280+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:42:14.185280+0000","last_undegraded":"2024-08-17T17:42:14.185280+0000","last_fullsized":"2024-08-17T17:42:14.185280+0000","mapping_epoch":375,"log_start":"383'3000","ondisk_log_start":"383'3000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208256,"num_objects":5771,"num_object_clones":0,"num_object_copies":17313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10029,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":739,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.706586+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:42:05.706586+0000","last_peered":"2024-08-17T17:42:05.706586+0000","last_clean":"2024-08-17T17:42:05.706586+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:42:05.706586+0000","last_undegraded":"2024-08-17T17:42:05.706586+0000","last_fullsized":"2024-08-17T17:42:05.706586+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":993,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803035+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:42:00.803035+0000","last_peered":"2024-08-17T17:42:00.803035+0000","last_clean":"2024-08-17T17:42:00.803035+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:42:00.803035+0000","last_undegraded":"2024-08-17T17:42:00.803035+0000","last_fullsized":"2024-08-17T17:42:00.803035+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":646,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.888550+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:42:07.888550+0000","last_peered":"2024-08-17T17:42:07.888550+0000","last_clean":"2024-08-17T17:42:07.888550+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:42:07.888550+0000","last_undegraded":"2024-08-17T17:42:07.888550+0000","last_fullsized":"2024-08-17T17:42:07.888550+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":306,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903255+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:42:07.903255+0000","last_peered":"2024-08-17T17:42:07.903255+0000","last_clean":"2024-08-17T17:42:07.903255+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:42:07.903255+0000","last_undegraded":"2024-08-17T17:42:07.903255+0000","last_fullsized":"2024-08-17T17:42:07.903255+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"400'9951","reported_seq":10013,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.633920+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:42:15.633920+0000","last_peered":"2024-08-17T17:42:15.633920+0000","last_clean":"2024-08-17T17:42:15.633920+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:42:15.633920+0000","last_undegraded":"2024-08-17T17:42:15.633920+0000","last_fullsized":"2024-08-17T17:42:15.633920+0000","mapping_epoch":375,"log_start":"380'800","ondisk_log_start":"380'800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374013952,"num_objects":5707,"num_object_clones":0,"num_object_copies":17121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9951,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":463,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.887824+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:42:07.887824+0000","last_peered":"2024-08-17T17:42:07.887824+0000","last_clean":"2024-08-17T17:42:07.887824+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:42:07.887824+0000","last_undegraded":"2024-08-17T17:42:07.887824+0000","last_fullsized":"2024-08-17T17:42:07.887824+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":779,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.805718+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:42:00.805718+0000","last_peered":"2024-08-17T17:42:00.805718+0000","last_clean":"2024-08-17T17:42:00.805718+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:42:00.805718+0000","last_undegraded":"2024-08-17T17:42:00.805718+0000","last_fullsized":"2024-08-17T17:42:00.805718+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":667,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884847+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:42:07.884847+0000","last_peered":"2024-08-17T17:42:07.884847+0000","last_clean":"2024-08-17T17:42:07.884847+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:42:07.884847+0000","last_undegraded":"2024-08-17T17:42:07.884847+0000","last_fullsized":"2024-08-17T17:42:07.884847+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":664,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882728+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:42:07.882728+0000","last_peered":"2024-08-17T17:42:07.882728+0000","last_clean":"2024-08-17T17:42:07.882728+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:42:07.882728+0000","last_undegraded":"2024-08-17T17:42:07.882728+0000","last_fullsized":"2024-08-17T17:42:07.882728+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"400'9935","reported_seq":9970,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.038964+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:42:16.038964+0000","last_peered":"2024-08-17T17:42:16.038964+0000","last_clean":"2024-08-17T17:42:16.038964+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:42:16.038964+0000","last_undegraded":"2024-08-17T17:42:16.038964+0000","last_fullsized":"2024-08-17T17:42:16.038964+0000","mapping_epoch":375,"log_start":"381'1100","ondisk_log_start":"381'1100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8835,"log_dups_size":0,"ondisk_log_size":8835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365494272,"num_objects":5577,"num_object_clones":0,"num_object_copies":16731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9935,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":464,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806520+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:42:00.806520+0000","last_peered":"2024-08-17T17:42:00.806520+0000","last_clean":"2024-08-17T17:42:00.806520+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:42:00.806520+0000","last_undegraded":"2024-08-17T17:42:00.806520+0000","last_fullsized":"2024-08-17T17:42:00.806520+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":469,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.885838+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:42:07.885838+0000","last_peered":"2024-08-17T17:42:07.885838+0000","last_clean":"2024-08-17T17:42:07.885838+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:42:07.885838+0000","last_undegraded":"2024-08-17T17:42:07.885838+0000","last_fullsized":"2024-08-17T17:42:07.885838+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":657,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.889563+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:42:07.889563+0000","last_peered":"2024-08-17T17:42:07.889563+0000","last_clean":"2024-08-17T17:42:07.889563+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:42:07.889563+0000","last_undegraded":"2024-08-17T17:42:07.889563+0000","last_fullsized":"2024-08-17T17:42:07.889563+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":704,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.889157+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:42:07.889157+0000","last_peered":"2024-08-17T17:42:07.889157+0000","last_clean":"2024-08-17T17:42:07.889157+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:42:07.889157+0000","last_undegraded":"2024-08-17T17:42:07.889157+0000","last_fullsized":"2024-08-17T17:42:07.889157+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"400'9926","reported_seq":9988,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.661976+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:42:15.661976+0000","last_peered":"2024-08-17T17:42:15.661976+0000","last_clean":"2024-08-17T17:42:15.661976+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:42:15.661976+0000","last_undegraded":"2024-08-17T17:42:15.661976+0000","last_fullsized":"2024-08-17T17:42:15.661976+0000","mapping_epoch":375,"log_start":"380'800","ondisk_log_start":"380'800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369754112,"num_objects":5642,"num_object_clones":0,"num_object_copies":16926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9926,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":416,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803972+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:42:00.803972+0000","last_peered":"2024-08-17T17:42:00.803972+0000","last_clean":"2024-08-17T17:42:00.803972+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:42:00.803972+0000","last_undegraded":"2024-08-17T17:42:00.803972+0000","last_fullsized":"2024-08-17T17:42:00.803972+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1116,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.805803+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:42:00.805803+0000","last_peered":"2024-08-17T17:42:00.805803+0000","last_clean":"2024-08-17T17:42:00.805803+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:42:00.805803+0000","last_undegraded":"2024-08-17T17:42:00.805803+0000","last_fullsized":"2024-08-17T17:42:00.805803+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":482,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884342+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:42:07.884342+0000","last_peered":"2024-08-17T17:42:07.884342+0000","last_clean":"2024-08-17T17:42:07.884342+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:42:07.884342+0000","last_undegraded":"2024-08-17T17:42:07.884342+0000","last_fullsized":"2024-08-17T17:42:07.884342+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":47,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.804130+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:42:00.804130+0000","last_peered":"2024-08-17T17:42:00.804130+0000","last_clean":"2024-08-17T17:42:00.804130+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:42:00.804130+0000","last_undegraded":"2024-08-17T17:42:00.804130+0000","last_fullsized":"2024-08-17T17:42:00.804130+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"400'9866","reported_seq":9898,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.034763+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:42:16.034763+0000","last_peered":"2024-08-17T17:42:16.034763+0000","last_clean":"2024-08-17T17:42:16.034763+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:42:16.034763+0000","last_undegraded":"2024-08-17T17:42:16.034763+0000","last_fullsized":"2024-08-17T17:42:16.034763+0000","mapping_epoch":375,"log_start":"381'1000","ondisk_log_start":"381'1000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369754112,"num_objects":5642,"num_object_clones":0,"num_object_copies":16926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9866,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":460,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.888286+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:42:07.888286+0000","last_peered":"2024-08-17T17:42:07.888286+0000","last_clean":"2024-08-17T17:42:07.888286+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:42:07.888286+0000","last_undegraded":"2024-08-17T17:42:07.888286+0000","last_fullsized":"2024-08-17T17:42:07.888286+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803661+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:42:00.803661+0000","last_peered":"2024-08-17T17:42:00.803661+0000","last_clean":"2024-08-17T17:42:00.803661+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:42:00.803661+0000","last_undegraded":"2024-08-17T17:42:00.803661+0000","last_fullsized":"2024-08-17T17:42:00.803661+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":949,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.886715+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:42:07.886715+0000","last_peered":"2024-08-17T17:42:07.886715+0000","last_clean":"2024-08-17T17:42:07.886715+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:42:07.886715+0000","last_undegraded":"2024-08-17T17:42:07.886715+0000","last_fullsized":"2024-08-17T17:42:07.886715+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":673,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.894159+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:42:07.894159+0000","last_peered":"2024-08-17T17:42:07.894159+0000","last_clean":"2024-08-17T17:42:07.894159+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:42:07.894159+0000","last_undegraded":"2024-08-17T17:42:07.894159+0000","last_fullsized":"2024-08-17T17:42:07.894159+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"400'10080","reported_seq":10114,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.729281+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:42:15.729281+0000","last_peered":"2024-08-17T17:42:15.729281+0000","last_clean":"2024-08-17T17:42:15.729281+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:42:15.729281+0000","last_undegraded":"2024-08-17T17:42:15.729281+0000","last_fullsized":"2024-08-17T17:42:15.729281+0000","mapping_epoch":375,"log_start":"383'3103","ondisk_log_start":"383'3103","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373293056,"num_objects":5696,"num_object_clones":0,"num_object_copies":17088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":466,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882388+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:42:07.882388+0000","last_peered":"2024-08-17T17:42:07.882388+0000","last_clean":"2024-08-17T17:42:07.882388+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:42:07.882388+0000","last_undegraded":"2024-08-17T17:42:07.882388+0000","last_fullsized":"2024-08-17T17:42:07.882388+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":670,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.886629+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:42:07.886629+0000","last_peered":"2024-08-17T17:42:07.886629+0000","last_clean":"2024-08-17T17:42:07.886629+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:42:07.886629+0000","last_undegraded":"2024-08-17T17:42:07.886629+0000","last_fullsized":"2024-08-17T17:42:07.886629+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":969,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883200+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:42:07.883200+0000","last_peered":"2024-08-17T17:42:07.883200+0000","last_clean":"2024-08-17T17:42:07.883200+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:42:07.883200+0000","last_undegraded":"2024-08-17T17:42:07.883200+0000","last_fullsized":"2024-08-17T17:42:07.883200+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":667,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882584+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:42:07.882584+0000","last_peered":"2024-08-17T17:42:07.882584+0000","last_clean":"2024-08-17T17:42:07.882584+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:42:07.882584+0000","last_undegraded":"2024-08-17T17:42:07.882584+0000","last_fullsized":"2024-08-17T17:42:07.882584+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"400'10058","reported_seq":10094,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.717207+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:42:15.717207+0000","last_peered":"2024-08-17T17:42:15.717207+0000","last_clean":"2024-08-17T17:42:15.717207+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:42:15.717207+0000","last_undegraded":"2024-08-17T17:42:15.717207+0000","last_fullsized":"2024-08-17T17:42:15.717207+0000","mapping_epoch":375,"log_start":"383'3014","ondisk_log_start":"383'3014","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375848984,"num_objects":5736,"num_object_clones":0,"num_object_copies":17208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5736,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10058,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":416,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807428+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:42:00.807428+0000","last_peered":"2024-08-17T17:42:00.807428+0000","last_clean":"2024-08-17T17:42:00.807428+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:42:00.807428+0000","last_undegraded":"2024-08-17T17:42:00.807428+0000","last_fullsized":"2024-08-17T17:42:00.807428+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":660,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.888554+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:42:07.888554+0000","last_peered":"2024-08-17T17:42:07.888554+0000","last_clean":"2024-08-17T17:42:07.888554+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:42:07.888554+0000","last_undegraded":"2024-08-17T17:42:07.888554+0000","last_fullsized":"2024-08-17T17:42:07.888554+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1180,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.896186+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:42:07.896186+0000","last_peered":"2024-08-17T17:42:07.896186+0000","last_clean":"2024-08-17T17:42:07.896186+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:42:07.896186+0000","last_undegraded":"2024-08-17T17:42:07.896186+0000","last_fullsized":"2024-08-17T17:42:07.896186+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":658,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884122+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:42:07.884122+0000","last_peered":"2024-08-17T17:42:07.884122+0000","last_clean":"2024-08-17T17:42:07.884122+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:42:07.884122+0000","last_undegraded":"2024-08-17T17:42:07.884122+0000","last_fullsized":"2024-08-17T17:42:07.884122+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"400'10033","reported_seq":10095,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:14.250619+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:42:14.250619+0000","last_peered":"2024-08-17T17:42:14.250619+0000","last_clean":"2024-08-17T17:42:14.250619+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:42:14.250619+0000","last_undegraded":"2024-08-17T17:42:14.250619+0000","last_fullsized":"2024-08-17T17:42:14.250619+0000","mapping_epoch":375,"log_start":"383'3000","ondisk_log_start":"383'3000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388169728,"num_objects":5923,"num_object_clones":0,"num_object_copies":17769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10033,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":460,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883323+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:42:07.883323+0000","last_peered":"2024-08-17T17:42:07.883323+0000","last_clean":"2024-08-17T17:42:07.883323+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:42:07.883323+0000","last_undegraded":"2024-08-17T17:42:07.883323+0000","last_fullsized":"2024-08-17T17:42:07.883323+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":679,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882483+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:42:07.882483+0000","last_peered":"2024-08-17T17:42:07.882483+0000","last_clean":"2024-08-17T17:42:07.882483+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:42:07.882483+0000","last_undegraded":"2024-08-17T17:42:07.882483+0000","last_fullsized":"2024-08-17T17:42:07.882483+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":32,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808297+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:42:00.808297+0000","last_peered":"2024-08-17T17:42:00.808297+0000","last_clean":"2024-08-17T17:42:00.808297+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:42:00.808297+0000","last_undegraded":"2024-08-17T17:42:00.808297+0000","last_fullsized":"2024-08-17T17:42:00.808297+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":343,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806633+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:42:00.806633+0000","last_peered":"2024-08-17T17:42:00.806633+0000","last_clean":"2024-08-17T17:42:00.806633+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:42:00.806633+0000","last_undegraded":"2024-08-17T17:42:00.806633+0000","last_fullsized":"2024-08-17T17:42:00.806633+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":306,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807389+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:42:00.807389+0000","last_peered":"2024-08-17T17:42:00.807389+0000","last_clean":"2024-08-17T17:42:00.807389+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:42:00.807389+0000","last_undegraded":"2024-08-17T17:42:00.807389+0000","last_fullsized":"2024-08-17T17:42:00.807389+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":316,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.808199+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:42:00.808199+0000","last_peered":"2024-08-17T17:42:00.808199+0000","last_clean":"2024-08-17T17:42:00.808199+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:42:00.808199+0000","last_undegraded":"2024-08-17T17:42:00.808199+0000","last_fullsized":"2024-08-17T17:42:00.808199+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":655,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.894007+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:42:07.894007+0000","last_peered":"2024-08-17T17:42:07.894007+0000","last_clean":"2024-08-17T17:42:07.894007+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:42:07.894007+0000","last_undegraded":"2024-08-17T17:42:07.894007+0000","last_fullsized":"2024-08-17T17:42:07.894007+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":401,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.810782+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:42:00.810782+0000","last_peered":"2024-08-17T17:42:00.810782+0000","last_clean":"2024-08-17T17:42:00.810782+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:42:00.810782+0000","last_undegraded":"2024-08-17T17:42:00.810782+0000","last_fullsized":"2024-08-17T17:42:00.810782+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":184,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803628+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:42:00.803628+0000","last_peered":"2024-08-17T17:42:00.803628+0000","last_clean":"2024-08-17T17:42:00.803628+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:42:00.803628+0000","last_undegraded":"2024-08-17T17:42:00.803628+0000","last_fullsized":"2024-08-17T17:42:00.803628+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":623,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.885632+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:42:07.885632+0000","last_peered":"2024-08-17T17:42:07.885632+0000","last_clean":"2024-08-17T17:42:07.885632+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:42:07.885632+0000","last_undegraded":"2024-08-17T17:42:07.885632+0000","last_fullsized":"2024-08-17T17:42:07.885632+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":681,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882421+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:42:07.882421+0000","last_peered":"2024-08-17T17:42:07.882421+0000","last_clean":"2024-08-17T17:42:07.882421+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:42:07.882421+0000","last_undegraded":"2024-08-17T17:42:07.882421+0000","last_fullsized":"2024-08-17T17:42:07.882421+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":672,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884075+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:42:07.884075+0000","last_peered":"2024-08-17T17:42:07.884075+0000","last_clean":"2024-08-17T17:42:07.884075+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:42:07.884075+0000","last_undegraded":"2024-08-17T17:42:07.884075+0000","last_fullsized":"2024-08-17T17:42:07.884075+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806278+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:42:00.806278+0000","last_peered":"2024-08-17T17:42:00.806278+0000","last_clean":"2024-08-17T17:42:00.806278+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:42:00.806278+0000","last_undegraded":"2024-08-17T17:42:00.806278+0000","last_fullsized":"2024-08-17T17:42:00.806278+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":676,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.884309+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:42:07.884309+0000","last_peered":"2024-08-17T17:42:07.884309+0000","last_clean":"2024-08-17T17:42:07.884309+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:42:07.884309+0000","last_undegraded":"2024-08-17T17:42:07.884309+0000","last_fullsized":"2024-08-17T17:42:07.884309+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":654,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882652+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:42:07.882652+0000","last_peered":"2024-08-17T17:42:07.882652+0000","last_clean":"2024-08-17T17:42:07.882652+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:42:07.882652+0000","last_undegraded":"2024-08-17T17:42:07.882652+0000","last_fullsized":"2024-08-17T17:42:07.882652+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":658,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.886822+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:42:07.886822+0000","last_peered":"2024-08-17T17:42:07.886822+0000","last_clean":"2024-08-17T17:42:07.886822+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:42:07.886822+0000","last_undegraded":"2024-08-17T17:42:07.886822+0000","last_fullsized":"2024-08-17T17:42:07.886822+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":673,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883348+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:42:07.883348+0000","last_peered":"2024-08-17T17:42:07.883348+0000","last_clean":"2024-08-17T17:42:07.883348+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:42:07.883348+0000","last_undegraded":"2024-08-17T17:42:07.883348+0000","last_fullsized":"2024-08-17T17:42:07.883348+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":290,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806981+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:42:00.806981+0000","last_peered":"2024-08-17T17:42:00.806981+0000","last_clean":"2024-08-17T17:42:00.806981+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:42:00.806981+0000","last_undegraded":"2024-08-17T17:42:00.806981+0000","last_fullsized":"2024-08-17T17:42:00.806981+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.888968+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:42:07.888968+0000","last_peered":"2024-08-17T17:42:07.888968+0000","last_clean":"2024-08-17T17:42:07.888968+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:42:07.888968+0000","last_undegraded":"2024-08-17T17:42:07.888968+0000","last_fullsized":"2024-08-17T17:42:07.888968+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":48,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.881696+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:42:07.881696+0000","last_peered":"2024-08-17T17:42:07.881696+0000","last_clean":"2024-08-17T17:42:07.881696+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:42:07.881696+0000","last_undegraded":"2024-08-17T17:42:07.881696+0000","last_fullsized":"2024-08-17T17:42:07.881696+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":290,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.807317+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:42:00.807317+0000","last_peered":"2024-08-17T17:42:00.807317+0000","last_clean":"2024-08-17T17:42:00.807317+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:42:00.807317+0000","last_undegraded":"2024-08-17T17:42:00.807317+0000","last_fullsized":"2024-08-17T17:42:00.807317+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":188,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.804251+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:42:00.804251+0000","last_peered":"2024-08-17T17:42:00.804251+0000","last_clean":"2024-08-17T17:42:00.804251+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:42:00.804251+0000","last_undegraded":"2024-08-17T17:42:00.804251+0000","last_fullsized":"2024-08-17T17:42:00.804251+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":290,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806907+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:42:00.806907+0000","last_peered":"2024-08-17T17:42:00.806907+0000","last_clean":"2024-08-17T17:42:00.806907+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:42:00.806907+0000","last_undegraded":"2024-08-17T17:42:00.806907+0000","last_fullsized":"2024-08-17T17:42:00.806907+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":198,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.803793+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:42:00.803793+0000","last_peered":"2024-08-17T17:42:00.803793+0000","last_clean":"2024-08-17T17:42:00.803793+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:42:00.803793+0000","last_undegraded":"2024-08-17T17:42:00.803793+0000","last_fullsized":"2024-08-17T17:42:00.803793+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":678,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.885965+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:42:07.885965+0000","last_peered":"2024-08-17T17:42:07.885965+0000","last_clean":"2024-08-17T17:42:07.885965+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:42:07.885965+0000","last_undegraded":"2024-08-17T17:42:07.885965+0000","last_fullsized":"2024-08-17T17:42:07.885965+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":679,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883761+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:42:07.883761+0000","last_peered":"2024-08-17T17:42:07.883761+0000","last_clean":"2024-08-17T17:42:07.883761+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:42:07.883761+0000","last_undegraded":"2024-08-17T17:42:07.883761+0000","last_fullsized":"2024-08-17T17:42:07.883761+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":306,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:42:00.806422+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:42:00.806422+0000","last_peered":"2024-08-17T17:42:00.806422+0000","last_clean":"2024-08-17T17:42:00.806422+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:42:00.806422+0000","last_undegraded":"2024-08-17T17:42:00.806422+0000","last_fullsized":"2024-08-17T17:42:00.806422+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":698,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.882310+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:42:07.882310+0000","last_peered":"2024-08-17T17:42:07.882310+0000","last_clean":"2024-08-17T17:42:07.882310+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:42:07.882310+0000","last_undegraded":"2024-08-17T17:42:07.882310+0000","last_fullsized":"2024-08-17T17:42:07.882310+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5971443736,"num_objects":91118,"num_object_clones":0,"num_object_copies":273354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1556,"num_objects_misplaced":5725,"num_objects_unfound":0,"num_objects_dirty":91118,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158868,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":518,"num_bytes_recovered":33947648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18035716096,"data_stored":18035703880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126413,"ondisk_log_size":126413,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201745,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3708384,"kb_used_data":2941400,"kb_used_omap":1,"kb_used_meta":766974,"kb_avail":90036768,"statfs":{"total":95995035648,"available":92197650432,"internally_reserved":0,"allocated":3011993600,"data_stored":3010019512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":785381835},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922241,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455372,"kb_used_data":1110980,"kb_used_omap":7,"kb_used_meta":344376,"kb_avail":92289780,"statfs":{"total":95995035648,"available":94504734720,"internally_reserved":0,"allocated":1137643520,"data_stored":1135688854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":352641718},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085765,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3739988,"kb_used_data":3287820,"kb_used_omap":7,"kb_used_meta":452088,"kb_avail":90005164,"statfs":{"total":95995035648,"available":92165287936,"internally_reserved":0,"allocated":3366727680,"data_stored":3364762123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":462938553},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986655,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1815704,"kb_used_data":1116816,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":91929448,"statfs":{"total":95995035648,"available":94135754752,"internally_reserved":0,"allocated":1143619584,"data_stored":1141653602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":715651526},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435546,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3865096,"kb_used_data":2967104,"kb_used_omap":5,"kb_used_meta":897978,"kb_avail":89880056,"statfs":{"total":95995035648,"available":92037177344,"internally_reserved":0,"allocated":3038314496,"data_stored":3036360642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":919529545},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599073,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1891912,"kb_used_data":1080000,"kb_used_omap":7,"kb_used_meta":811896,"kb_avail":91853240,"statfs":{"total":95995035648,"available":94057717760,"internally_reserved":0,"allocated":1105920000,"data_stored":1103965495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":831381677},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762599,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4521536,"kb_used_data":3754040,"kb_used_omap":7,"kb_used_meta":767480,"kb_avail":89223616,"statfs":{"total":95995035648,"available":91364982784,"internally_reserved":0,"allocated":3844136960,"data_stored":3842189257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":785899937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926125,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2266636,"kb_used_data":1476932,"kb_used_omap":7,"kb_used_meta":789688,"kb_avail":91478516,"statfs":{"total":95995035648,"available":93674000384,"internally_reserved":0,"allocated":1512378368,"data_stored":1510420661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":808641116},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1492844544,"data_stored":1492844544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3827240960,"data_stored":3827236888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1097072640,"data_stored":1097072640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3028156416,"data_stored":3028156416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1132920832,"data_stored":1132920832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3341357056,"data_stored":3341352984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1125187584,"data_stored":1125187584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2990936064,"data_stored":2990931992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:19.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:42:19.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:19.935 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:19.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:20.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.113 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:20.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:20.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:20.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:20.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:20.378 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:20.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:20.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:20.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:20.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:20 smithi175 ceph-mon[101434]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:42:20.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:20 smithi064 ceph-mon[104516]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:42:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:20 smithi064 ceph-mon[109246]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 1556/273438 objects degraded (0.569%); 5725/273438 objects misplaced (2.094%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:42:20.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:20.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:20.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:20.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:20.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:42:20.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:20.990 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:21.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:21.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:21.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:21.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:21.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:21.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:21.470 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:21.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:21.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:21.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:21.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:21.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:21.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:21.900 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:22.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:22.031 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:22.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:22.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:22.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:22.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:22.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:22.435 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:22.537 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:22.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:22.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:22 smithi175 ceph-mon[101434]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 1765/262479 objects degraded (0.672%); 5460/262479 objects misplaced (2.080%); 949 KiB/s, 14 objects/s recovering 2024-08-17T17:42:22.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:22.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:22 smithi064 ceph-mon[104516]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 1765/262479 objects degraded (0.672%); 5460/262479 objects misplaced (2.080%); 949 KiB/s, 14 objects/s recovering 2024-08-17T17:42:22.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:22 smithi064 ceph-mon[109246]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 1765/262479 objects degraded (0.672%); 5460/262479 objects misplaced (2.080%); 949 KiB/s, 14 objects/s recovering 2024-08-17T17:42:22.717 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:22.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:22.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:22.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:23.014 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:23.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.162 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:23.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:23.263 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:23.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:23.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:23.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:23.433 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:42:23.483 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:23 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3311311737' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.483 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:23 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.484 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1765/262479 objects degraded (0.672%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:23.484 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:42:23.484 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:42:23.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164866 8.14 8:2e9a04ab:::benchmark_data_smithi064_242541_object42373:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.040908+0000", 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 299.44061118000002, 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88232739100000002, 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164866 2024-08-17T17:42:23.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040908+0000", 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040908+0000", 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040899+0000", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999909 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040920+0000", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0877e-05 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040956+0000", 2024-08-17T17:42:23.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6507999999999997e-05 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040970+0000", 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3254999999999999e-05 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221761+0000", 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18079186799999999 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221769+0000", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0299999999999996e-06 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.923158+0000", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.701389068 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.923235+0000", 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7786000000000004e-05 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.489 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164875 8.14 8:2a519829:::benchmark_data_smithi064_242541_object42382:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.040980+0000", 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 299.44053898999999, 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.88256932099999996, 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.490 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164875 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040980+0000", 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.491 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040980+0000", 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040980+0000", 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040982+0000", 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5639999999999999e-06 2024-08-17T17:42:23.492 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040988+0000", 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.767e-06 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040993+0000", 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5360000000000003e-06 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.493 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221786+0000", 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.180793077 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221791+0000", 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1379999999999998e-06 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.923497+0000", 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70170596799999996 2024-08-17T17:42:23.494 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.923549+0000", 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2707999999999997e-05 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164877 8.4 8:22d88133:::benchmark_data_smithi064_242541_object42384:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.040996+0000", 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 299.44052282000001, 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.83601308699999999, 2024-08-17T17:42:23.495 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164877 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040996+0000", 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.496 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040996+0000", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040996+0000", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.040998+0000", 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.153e-06 2024-08-17T17:42:23.497 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041003+0000", 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6360000000000002e-06 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041007+0000", 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9580000000000001e-06 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.498 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221009+0000", 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18000242999999999 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.221018+0000", 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9570000000000008e-06 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.876930+0000", 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.65591189699999997 2024-08-17T17:42:23.499 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.877009+0000", 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9439999999999998e-05 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164888 8.18 8:1d423e94:::benchmark_data_smithi064_242541_object42395:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.041010+0000", 2024-08-17T17:42:23.500 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 299.44050954199997, 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.90320150799999999, 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164888 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041010+0000", 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.501 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041010+0000", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041009+0000", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041012+0000", 2024-08-17T17:42:23.502 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1490000000000001e-06 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041016+0000", 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6099999999999999e-06 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041020+0000", 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6610000000000002e-06 2024-08-17T17:42:23.503 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.233236+0000", 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.192216256 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.233243+0000", 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2710000000000003e-06 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.504 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.937940+0000", 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70469627000000001 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.938054+0000", 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011431599999999999 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.938420+0000", 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036577799999999998 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.505 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940022+0000", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0016019540000000001 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944054+0000", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0040324460000000003 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944128+0000", 2024-08-17T17:42:23.506 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3800999999999994e-05 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944174+0000", 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6215999999999999e-05 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944211+0000", 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7039999999999998e-05 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.507 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164893 8.18 8:1a68af6d:::benchmark_data_smithi064_242541_object42400:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.041023+0000", 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 299.44049579199998, 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.91773043499999996, 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164893 2024-08-17T17:42:23.508 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041023+0000", 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041023+0000", 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.509 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041023+0000", 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041025+0000", 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8619999999999999e-06 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041030+0000", 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2979999999999999e-06 2024-08-17T17:42:23.510 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.041034+0000", 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8260000000000003e-06 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.233259+0000", 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19222475 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.233264+0000", 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1220000000000003e-06 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940088+0000", 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70682450799999996 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940201+0000", 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112958 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.940468+0000", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00026656299999999999 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944097+0000", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0036294510000000001 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.944261+0000", 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000163516 2024-08-17T17:42:23.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.958697+0000", 2024-08-17T17:42:23.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.014435686 2024-08-17T17:42:23.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.958728+0000", 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1673e-05 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.958754+0000", 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5483999999999999e-05 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.515 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216049 8.20 8:05efd7af:::benchmark_data_smithi064_242541_object93556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:43.532877+0000", 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.948642495, 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.252758644, 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.516 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216049 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532877+0000", 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532877+0000", 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532875+0000", 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532883+0000", 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0609999999999996e-06 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532894+0000", 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1228e-05 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532900+0000", 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6219999999999997e-06 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532952+0000", 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2228000000000002e-05 2024-08-17T17:42:23.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.532963+0000", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1235e-05 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317840+0000", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.78487677199999994 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317851+0000", 2024-08-17T17:42:23.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0647e-05 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.782049+0000", 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4641983299999999 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.782138+0000", 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8770000000000006e-05 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.782328+0000", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018993699999999999 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785191+0000", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002863184 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785248+0000", 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6598000000000003e-05 2024-08-17T17:42:23.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785403+0000", 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015559999999999999 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785459+0000", 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6187000000000003e-05 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785516+0000", 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6350000000000001e-05 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785635+0000", 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011963 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216219 8.20 8:058920d5:::benchmark_data_smithi064_242541_object93726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:42:23.524 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:43.736853+0000", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.74466637800001, 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0493130480000001, 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216219 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736853+0000", 2024-08-17T17:42:23.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736853+0000", 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736852+0000", 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736858+0000", 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1550000000000004e-06 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736870+0000", 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2248e-05 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736876+0000", 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736917+0000", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0878999999999999e-05 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.736925+0000", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9559999999999997e-06 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317878+0000", 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.58095295199999997 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317887+0000", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9109999999999999e-06 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.783411+0000", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4655247650000001 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.783510+0000", 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8126000000000002e-05 2024-08-17T17:42:23.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.783684+0000", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017411799999999999 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785369+0000", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001685609 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785686+0000", 2024-08-17T17:42:23.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00031642699999999998 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785890+0000", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00020444500000000001 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786006+0000", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011616399999999999 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786053+0000", 2024-08-17T17:42:23.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6986e-05 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786166+0000", 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112466 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216231 8.20 8:075619ba:::benchmark_data_smithi064_242541_object93738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:43.749565+0000", 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.731954325, 2024-08-17T17:42:23.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0367276259999998, 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216231 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749565+0000", 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749565+0000", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749564+0000", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749571+0000", 2024-08-17T17:42:23.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.131e-06 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749591+0000", 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9939999999999999e-05 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749597+0000", 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2389999999999999e-06 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749675+0000", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.763e-05 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.749684+0000", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4709999999999996e-06 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317911+0000", 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56822628100000006 2024-08-17T17:42:23.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317918+0000", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4909999999999999e-06 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.784438+0000", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4665193839999999 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.784505+0000", 2024-08-17T17:42:23.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7510999999999998e-05 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.784642+0000", 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013657299999999999 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785648+0000", 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0010061759999999999 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785935+0000", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00028766700000000002 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.785966+0000", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0414000000000001e-05 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786204+0000", 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023837700000000001 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786232+0000", 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7321000000000002e-05 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786292+0000", 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0843e-05 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.540 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216321 8.20 8:06752671:::benchmark_data_smithi064_242541_object93828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:43.879369+0000", 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.602149965, 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.917371972, 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216321 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879369+0000", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879369+0000", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879368+0000", 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879373+0000", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5759999999999999e-06 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879384+0000", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.112e-05 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879389+0000", 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9350000000000002e-06 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879427+0000", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8696999999999999e-05 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.879435+0000", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.277e-06 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317943+0000", 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.438508548 2024-08-17T17:42:23.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.317950+0000", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2989999999999999e-06 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786325+0000", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.468374888 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786443+0000", 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011762 2024-08-17T17:42:23.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.786627+0000", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000184334 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.789918+0000", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032907040000000002 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.789975+0000", 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7420999999999998e-05 2024-08-17T17:42:23.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796414+0000", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0064385639999999999 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796529+0000", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011459 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796580+0000", 2024-08-17T17:42:23.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1570000000000003e-05 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796741+0000", 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016101599999999999 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216352 8.20 8:075eb6b7:::benchmark_data_smithi064_242541_object93859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:43.986423+0000", 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.49509575100001, 2024-08-17T17:42:23.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.810543421, 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216352 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986423+0000", 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986423+0000", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986422+0000", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986427+0000", 2024-08-17T17:42:23.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7230000000000004e-06 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986438+0000", 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0886999999999999e-05 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986443+0000", 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2140000000000004e-06 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986478+0000", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5126999999999998e-05 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:43.986485+0000", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1119999999999998e-06 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.318091+0000", 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.33160596799999997 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.318101+0000", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7450000000000007e-06 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.787511+0000", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4694096320000001 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.787630+0000", 2024-08-17T17:42:23.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000119558 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.787905+0000", 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027520799999999998 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.789966+0000", 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0020609589999999998 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.789982+0000", 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5174000000000001e-05 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796789+0000", 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0068072150000000001 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796824+0000", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4969999999999999e-05 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796861+0000", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7568999999999999e-05 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796967+0000", 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105487 2024-08-17T17:42:23.556 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.557 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.557 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:23.558 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216360 8.20 8:042b45fd:::benchmark_data_smithi064_242541_object93867:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.558 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.369314+0000", 2024-08-17T17:42:23.558 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.112205332, 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4278944, 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216360 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369314+0000", 2024-08-17T17:42:23.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369314+0000", 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369313+0000", 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369318+0000", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6169999999999998e-06 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369330+0000", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1365999999999999e-05 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369358+0000", 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8107e-05 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369425+0000", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7049000000000001e-05 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.369447+0000", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2408e-05 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.788638+0000", 2024-08-17T17:42:23.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4191907239999999 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.788753+0000", 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011469200000000001 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.788949+0000", 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019635200000000001 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796350+0000", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0074015979999999997 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796361+0000", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0784e-05 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797037+0000", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00067599999999999995 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797080+0000", 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2867999999999997e-05 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797119+0000", 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8980000000000003e-05 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797208+0000", 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9135000000000006e-05 2024-08-17T17:42:23.565 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216441 8.20 8:0485de1b:::benchmark_data_smithi064_242541_object93948:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.438223+0000", 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 219.043296599, 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.359338811, 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.566 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216441 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438223+0000", 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438223+0000", 2024-08-17T17:42:23.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438221+0000", 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438226+0000", 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8199999999999996e-06 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438237+0000", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1362999999999999e-05 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438244+0000", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6440000000000003e-06 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438295+0000", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1276000000000002e-05 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.438304+0000", 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4800000000000007e-06 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.790017+0000", 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3517124229999999 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.790087+0000", 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0564999999999999e-05 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.790237+0000", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014993800000000001 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796469+0000", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0062319760000000002 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.796480+0000", 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0723000000000001e-05 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797327+0000", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00084662400000000003 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797383+0000", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6539000000000002e-05 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797436+0000", 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2741000000000002e-05 2024-08-17T17:42:23.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797561+0000", 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000125346 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216608 8.20 8:05d43480:::benchmark_data_smithi064_242541_object94115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.578930+0000", 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.902589205, 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2268523120000001, 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.573 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216608 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578930+0000", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578930+0000", 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578929+0000", 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578935+0000", 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.635e-06 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578952+0000", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6765999999999998e-05 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.578960+0000", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3210000000000005e-06 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.579022+0000", 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1637999999999995e-05 2024-08-17T17:42:23.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.579032+0000", 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0169e-05 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797686+0000", 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2186542760000001 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.797791+0000", 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010481500000000001 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.798106+0000", 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00031508399999999998 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802445+0000", 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043390950000000003 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802523+0000", 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.716e-05 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802619+0000", 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6881999999999998e-05 2024-08-17T17:42:23.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805650+0000", 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030301289999999999 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805689+0000", 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9700999999999998e-05 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805782+0000", 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2964e-05 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216619 8.20 8:05689918:::benchmark_data_smithi064_242541_object94126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.588173+0000", 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.893345915, 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2178892050000001, 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.581 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216619 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588173+0000", 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588173+0000", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588172+0000", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588178+0000", 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3279999999999996e-06 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588188+0000", 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.048e-05 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588196+0000", 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7549999999999994e-06 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588257+0000", 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1643999999999995e-05 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.588269+0000", 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1343e-05 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.799519+0000", 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2112504930000001 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.799642+0000", 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000122564 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.799900+0000", 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025795800000000003 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802557+0000", 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026571070000000001 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802588+0000", 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0753000000000002e-05 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805820+0000", 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032325209999999999 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805901+0000", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1156999999999996e-05 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805949+0000", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8158999999999997e-05 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806062+0000", 2024-08-17T17:42:23.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000112979 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216751 8.20 8:07863277:::benchmark_data_smithi064_242541_object94258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.708045+0000", 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.77347443400001, 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1054581510000001, 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.589 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216751 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708045+0000", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708045+0000", 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708043+0000", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708048+0000", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708058+0000", 2024-08-17T17:42:23.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4730000000000006e-06 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708064+0000", 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8610000000000003e-06 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708142+0000", 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8714000000000006e-05 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.708153+0000", 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0543999999999999e-05 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802644+0000", 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.094491104 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.802754+0000", 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010992700000000001 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.803018+0000", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00026419900000000001 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805637+0000", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026187139999999999 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.805890+0000", 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000253253 2024-08-17T17:42:23.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806072+0000", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018214699999999999 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813359+0000", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0072863709999999998 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813400+0000", 2024-08-17T17:42:23.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1084999999999997e-05 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813503+0000", 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000103263 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216754 8.20 8:0586c3e1:::benchmark_data_smithi064_242541_object94261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.709016+0000", 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.772502723, 2024-08-17T17:42:23.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.104659222, 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216754 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709016+0000", 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709016+0000", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709016+0000", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709020+0000", 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4009999999999999e-06 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709030+0000", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0173000000000001e-05 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709036+0000", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2339999999999999e-06 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709081+0000", 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4702e-05 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.709089+0000", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3289999999999994e-06 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.804151+0000", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.095062024 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.804256+0000", 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104472 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.804441+0000", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018508399999999999 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806077+0000", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0016363720000000001 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.808270+0000", 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021922869999999998 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.808304+0000", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4449999999999997e-05 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813539+0000", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0052347549999999998 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813579+0000", 2024-08-17T17:42:23.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0407999999999997e-05 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813676+0000", 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6329000000000006e-05 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:216980 8.20 8:0700dd33:::benchmark_data_smithi064_242541_object94487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:44.961089+0000", 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.52043010899999, 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.86584761700000001, 2024-08-17T17:42:23.603 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 216980 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961089+0000", 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961089+0000", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961088+0000", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961092+0000", 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1949999999999999e-06 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961101+0000", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4960000000000002e-06 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961106+0000", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.181e-06 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961155+0000", 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9085999999999997e-05 2024-08-17T17:42:23.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:44.961166+0000", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0783e-05 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806132+0000", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.84496532800000002 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806231+0000", 2024-08-17T17:42:23.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9936e-05 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.806458+0000", 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000226832 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813705+0000", 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0072466359999999999 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813723+0000", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7889000000000001e-05 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.816766+0000", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030431780000000001 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.826787+0000", 2024-08-17T17:42:23.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010020859 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.826837+0000", 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0176000000000002e-05 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.826937+0000", 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9622000000000003e-05 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:217051 8.20 8:04bffb9c:::benchmark_data_smithi064_242541_object94558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:45.048893+0000", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.43262648000001, 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.77820770699999997, 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 217051 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048893+0000", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048893+0000", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048891+0000", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048897+0000", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0410000000000003e-06 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048910+0000", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3148e-05 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048917+0000", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4300000000000003e-06 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048972+0000", 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5504000000000003e-05 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.048983+0000", 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0397000000000001e-05 2024-08-17T17:42:23.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.808732+0000", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.75974922499999997 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.808862+0000", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000130245 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.809151+0000", 2024-08-17T17:42:23.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00028893799999999998 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813741+0000", 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.004589625 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.813761+0000", 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.012e-05 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.817189+0000", 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0034280320000000001 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.826947+0000", 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0097586010000000004 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.826993+0000", 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5599999999999997e-05 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.827100+0000", 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010741 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:217075 8.20 8:0619eb74:::benchmark_data_smithi064_242541_object94582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:45.082869+0000", 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 218.398650535, 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.74442917600000003, 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.618 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 217075 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082869+0000", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082869+0000", 2024-08-17T17:42:23.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082867+0000", 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082873+0000", 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4559999999999997e-06 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082886+0000", 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3315999999999999e-05 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082893+0000", 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9079999999999998e-06 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082948+0000", 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4781000000000003e-05 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.082960+0000", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1497e-05 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.810450+0000", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.72749040200000004 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.811897+0000", 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0014464790000000001 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,3,4", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.812173+0000", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027602300000000002 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.821888+0000", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0097150910000000004 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.821922+0000", 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4505999999999998e-05 2024-08-17T17:42:23.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.827112+0000", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051897430000000001 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.827158+0000", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6020000000000003e-05 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.827198+0000", 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9631000000000001e-05 2024-08-17T17:42:23.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:45.827298+0000", 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000100265 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:220949 8.20 8:064dd08b:::benchmark_data_smithi064_242541_object98456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:49.692445+0000", 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 213.78907448699999, 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.67096186999999996, 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:42:23.625 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 220949 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692445+0000", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692445+0000", 2024-08-17T17:42:23.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692444+0000", 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692449+0000", 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2429999999999996e-06 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:42:23.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692498+0000", 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8916999999999999e-05 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692505+0000", 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2660000000000004e-06 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692548+0000", 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3204000000000002e-05 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:49.692557+0000", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6899999999999998e-06 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.359743+0000", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.66718558900000002 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.359864+0000", 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001215 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.360070+0000", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000206262 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.363260+0000", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003189206 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.363268+0000", 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8429999999999999e-06 2024-08-17T17:42:23.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.363346+0000", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7016999999999998e-05 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.363374+0000", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8453999999999999e-05 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:50.363407+0000", 2024-08-17T17:42:23.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2574999999999999e-05 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:42:23.632 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:42:23.641 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3311311737' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1765/262479 objects degraded (0.672%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3311311737' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:23.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1765/262479 objects degraded (0.672%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:23.743 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:23.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:23.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:23.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:23.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:42:23.933 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1765/261000 objects degraded (0.676%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1727,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":401,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915895,"num_in_osds":8,"osd_in_since":1723915756,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":98},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":87000,"data_bytes":5704521725,"bytes_used":23033204736,"bytes_avail":744927080448,"bytes_total":767960285184,"degraded_objects":1765,"degraded_total":261000,"degraded_ratio":0.0067624521072796925,"misplaced_objects":5460,"misplaced_total":261000,"misplaced_ratio":0.020919540229885056,"recovering_objects_per_sec":9,"recovering_bytes_per_sec":627768,"recovering_keys_per_sec":0,"num_objects_recovered":115,"num_bytes_recovered":7536640,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":686},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-17T17:34:55.113164+0000","services":{}},"progress_events":{}} 2024-08-17T17:42:24.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:24.059 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:24.066 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:23] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:42:24.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:24.198 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:24.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:24.334 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:24.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:24.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:24.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:24.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:24.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:42:24.678 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:24 smithi175 ceph-mon[101434]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 1765/261000 objects degraded (0.676%); 5460/261000 objects misplaced (2.092%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:24.679 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:24 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:24.679 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:24 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:24.679 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:24 smithi175 ceph-mon[101434]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:42:24.679 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1438131127' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[104516]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 1765/261000 objects degraded (0.676%); 5460/261000 objects misplaced (2.092%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[104516]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:42:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1438131127' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[109246]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 1765/261000 objects degraded (0.676%); 5460/261000 objects misplaced (2.092%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[109246]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:42:24.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1438131127' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:24.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.750 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:24.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:24.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:24.889 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:24.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:25.037 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:25.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:25.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:25.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:25.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:25.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:25.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:25.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:25.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:25.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:25.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:25.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:25.757 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:25.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:25.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:25.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:25.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:26.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:26.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:26.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:26.144 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:26.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.281 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:26.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:26.287 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:42:26.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:26.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.455 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:26.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:26.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:26.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:26.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:26 smithi064 ceph-mon[104516]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1765/258042 objects degraded (0.684%); 5460/258042 objects misplaced (2.116%); 736 KiB/s, 11 objects/s recovering 2024-08-17T17:42:26.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:26 smithi064 ceph-mon[109246]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1765/258042 objects degraded (0.684%); 5460/258042 objects misplaced (2.116%); 736 KiB/s, 11 objects/s recovering 2024-08-17T17:42:26.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:26.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:26.752 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:26 smithi175 ceph-mon[101434]: pgmap v1244: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 1765/258042 objects degraded (0.684%); 5460/258042 objects misplaced (2.116%); 736 KiB/s, 11 objects/s recovering 2024-08-17T17:42:26.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:42:26.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:26.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:26.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:26.976 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:26.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:42:26.987 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:27.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:27.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:27.029 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:42:27.029 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:27.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:42:27.075 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:27.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:27.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:42:27.112 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:27.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:27.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:27.150 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:42:27.150 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:27.189 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:42:27.189 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:27.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:42:27.228 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:27.238 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:27.264 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:42:27.264 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:27.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:27.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:27.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:27.406 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:27.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:27.508 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:27.545 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:27.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:27.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:27.673 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4079777120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:27.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4079777120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:27.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4079777120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:27.689 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:27.716 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:27.716 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:27.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:27.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:27.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:27.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:28.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:28.106 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:28.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:28.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:28.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:28.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:28.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:42:28.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:28.565 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:28 smithi175 ceph-mon[101434]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1933/249441 objects degraded (0.775%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:28.667 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:28.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:28.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:28 smithi064 ceph-mon[104516]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:28.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1933/249441 objects degraded (0.775%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:28.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:28 smithi064 ceph-mon[109246]: pgmap v1245: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:28.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1933/249441 objects degraded (0.775%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:28.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:28.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:28.846 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:28.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:28.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:29.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:29.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:29.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:29.235 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:29.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:29.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:29.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.426 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:29.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:29.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:29.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:29.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:29.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:29.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:29.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.854 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:29.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:29.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:29.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:30.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:30.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:30.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:30.267 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:30.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.405 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:30.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:30.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:30.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:30.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:30.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:30.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:30.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:42:30.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:30 smithi064 ceph-mon[104516]: pgmap v1246: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:30.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:30 smithi064 ceph-mon[109246]: pgmap v1246: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:30.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:30.774 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:30 smithi175 ceph-mon[101434]: pgmap v1246: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 1933/249441 objects degraded (0.775%); 5225/249441 objects misplaced (2.095%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:42:30.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:30.908 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:30.918 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:30.918 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:30.936 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:31.038 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:31.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:31.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.222 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:31.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:31.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:31.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:31.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.489 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:31.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:31.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:31.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:31.683 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:42:31.684 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:42:31.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:31.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:31.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:31.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:32.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:32.050 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:32.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.178 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:32.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:32.261 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:32.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:32.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:32.425 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:32.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:32.477 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:42:32.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:42:32.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:32.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[104516]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 2096/238704 objects degraded (0.878%); 4990/238704 objects misplaced (2.090%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[104516]: from='client.35928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1912019139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[109246]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 2096/238704 objects degraded (0.878%); 4990/238704 objects misplaced (2.090%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:32.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[109246]: from='client.35928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:32.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1912019139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:32.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:32 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:32.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:32.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:32.809 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:32.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.899 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:32.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:32 smithi175 ceph-mon[101434]: pgmap v1247: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 2096/238704 objects degraded (0.878%); 4990/238704 objects misplaced (2.090%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:32 smithi175 ceph-mon[101434]: from='client.35928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1912019139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:32 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:33.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:33.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:33.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:33.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:33.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.300 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:33.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:33.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:33.442 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:33.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:33.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:33 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:33 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:33 smithi175 ceph-mon[101434]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:42:33.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:33 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2096/238704 objects degraded (0.878%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[104516]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:42:33.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2096/238704 objects degraded (0.878%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[109246]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:42:33.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2096/238704 objects degraded (0.878%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:42:33.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:33.731 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:33.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:33.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:33.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:33.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:34.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:34.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:34.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:34.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:33] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-08-17T17:42:34.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:34.283 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:34.385 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:34.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:34.493 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1247,"stamp":"2024-08-17T17:42:31.282688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5217458173,"num_objects":79568,"num_object_clones":0,"num_object_copies":238704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2096,"num_objects_misplaced":4990,"num_objects_unfound":0,"num_objects_dirty":79568,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":171354,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":834,"num_bytes_recovered":55122147,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129363,"ondisk_log_size":129363,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21224532,"kb_used_data":15658964,"kb_used_omap":46,"kb_used_meta":5565329,"kb_avail":728736684,"statfs":{"total":767960285184,"available":746226364416,"internally_reserved":0,"allocated":16034779136,"data_stored":16019073778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5698897139},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,3,3,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-519372800,"num_objects":-7925,"num_object_clones":0,"num_object_copies":-23775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":331,"num_objects_misplaced":-470,"num_objects_unfound":0,"num_objects_dirty":-7925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7925,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003326"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":663,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.452481+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:42:23.452481+0000","last_peered":"2024-08-17T17:42:23.452481+0000","last_clean":"2024-08-17T17:42:23.452481+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:42:23.452481+0000","last_undegraded":"2024-08-17T17:42:23.452481+0000","last_fullsized":"2024-08-17T17:42:23.452481+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":401,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878347+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:42:07.878347+0000","last_peered":"2024-08-17T17:42:07.878347+0000","last_clean":"2024-08-17T17:42:07.878347+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:42:07.878347+0000","last_undegraded":"2024-08-17T17:42:07.878347+0000","last_fullsized":"2024-08-17T17:42:07.878347+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":185,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875860+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:42:07.875860+0000","last_peered":"2024-08-17T17:42:07.875860+0000","last_clean":"2024-08-17T17:42:07.875860+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:42:07.875860+0000","last_undegraded":"2024-08-17T17:42:07.875860+0000","last_fullsized":"2024-08-17T17:42:07.875860+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":654,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448897+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:42:23.448897+0000","last_peered":"2024-08-17T17:42:23.448897+0000","last_clean":"2024-08-17T17:42:23.448897+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:42:23.448897+0000","last_undegraded":"2024-08-17T17:42:23.448897+0000","last_fullsized":"2024-08-17T17:42:23.448897+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":654,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448465+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:42:23.448465+0000","last_peered":"2024-08-17T17:42:23.448465+0000","last_clean":"2024-08-17T17:42:23.448465+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:42:23.448465+0000","last_undegraded":"2024-08-17T17:42:23.448465+0000","last_fullsized":"2024-08-17T17:42:23.448465+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":387,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878399+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:42:07.878399+0000","last_peered":"2024-08-17T17:42:07.878399+0000","last_clean":"2024-08-17T17:42:07.878399+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:42:07.878399+0000","last_undegraded":"2024-08-17T17:42:07.878399+0000","last_fullsized":"2024-08-17T17:42:07.878399+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":662,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.456191+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:42:23.456191+0000","last_peered":"2024-08-17T17:42:23.456191+0000","last_clean":"2024-08-17T17:42:23.456191+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:42:23.456191+0000","last_undegraded":"2024-08-17T17:42:23.456191+0000","last_fullsized":"2024-08-17T17:42:23.456191+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":335,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.896142+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:42:07.896142+0000","last_peered":"2024-08-17T17:42:07.896142+0000","last_clean":"2024-08-17T17:42:07.896142+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:42:07.896142+0000","last_undegraded":"2024-08-17T17:42:07.896142+0000","last_fullsized":"2024-08-17T17:42:07.896142+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"401'10631","reported_seq":12120,"reported_epoch":401,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:30.785026+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:42:30.785026+0000","last_peered":"2024-08-17T17:42:30.785026+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:42:30.785026+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"382'1500","ondisk_log_start":"382'1500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9131,"log_dups_size":0,"ondisk_log_size":9131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531968,"num_objects":5013,"num_object_clones":0,"num_object_copies":15039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2096,"num_objects_misplaced":4990,"num_objects_unfound":0,"num_objects_dirty":5013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10631,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":713,"num_bytes_recovered":46727168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":4990},{"shards":"0,5","objects":-2073}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":397,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878671+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:42:07.878671+0000","last_peered":"2024-08-17T17:42:07.878671+0000","last_clean":"2024-08-17T17:42:07.878671+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:42:07.878671+0000","last_undegraded":"2024-08-17T17:42:07.878671+0000","last_fullsized":"2024-08-17T17:42:07.878671+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":196,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875250+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:42:07.875250+0000","last_peered":"2024-08-17T17:42:07.875250+0000","last_clean":"2024-08-17T17:42:07.875250+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:42:07.875250+0000","last_undegraded":"2024-08-17T17:42:07.875250+0000","last_fullsized":"2024-08-17T17:42:07.875250+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"401'10428","reported_seq":10492,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:26.260575+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:42:26.260575+0000","last_peered":"2024-08-17T17:42:26.260575+0000","last_clean":"2024-08-17T17:42:26.260575+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:42:26.260575+0000","last_undegraded":"2024-08-17T17:42:26.260575+0000","last_fullsized":"2024-08-17T17:42:26.260575+0000","mapping_epoch":375,"log_start":"383'3200","ondisk_log_start":"383'3200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7228,"log_dups_size":0,"ondisk_log_size":7228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341573632,"num_objects":5212,"num_object_clones":0,"num_object_copies":15636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10428,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":398,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878649+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:42:07.878649+0000","last_peered":"2024-08-17T17:42:07.878649+0000","last_clean":"2024-08-17T17:42:07.878649+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:42:07.878649+0000","last_undegraded":"2024-08-17T17:42:07.878649+0000","last_fullsized":"2024-08-17T17:42:07.878649+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":293,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.902824+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:42:07.902824+0000","last_peered":"2024-08-17T17:42:07.902824+0000","last_clean":"2024-08-17T17:42:07.902824+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:42:07.902824+0000","last_undegraded":"2024-08-17T17:42:07.902824+0000","last_fullsized":"2024-08-17T17:42:07.902824+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"401'10728","reported_seq":10764,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.096862+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:42:31.096862+0000","last_peered":"2024-08-17T17:42:31.096862+0000","last_clean":"2024-08-17T17:42:31.096862+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:42:31.096862+0000","last_undegraded":"2024-08-17T17:42:31.096862+0000","last_fullsized":"2024-08-17T17:42:31.096862+0000","mapping_epoch":375,"log_start":"382'1900","ondisk_log_start":"382'1900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321650688,"num_objects":4908,"num_object_clones":0,"num_object_copies":14724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10728,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":415,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883178+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:42:07.883178+0000","last_peered":"2024-08-17T17:42:07.883178+0000","last_clean":"2024-08-17T17:42:07.883178+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:42:07.883178+0000","last_undegraded":"2024-08-17T17:42:07.883178+0000","last_fullsized":"2024-08-17T17:42:07.883178+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":316,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.444757+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:42:23.444757+0000","last_peered":"2024-08-17T17:42:23.444757+0000","last_clean":"2024-08-17T17:42:23.444757+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:42:23.444757+0000","last_undegraded":"2024-08-17T17:42:23.444757+0000","last_fullsized":"2024-08-17T17:42:23.444757+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"401'10735","reported_seq":10771,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.137750+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:42:31.137750+0000","last_peered":"2024-08-17T17:42:31.137750+0000","last_clean":"2024-08-17T17:42:31.137750+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:42:31.137750+0000","last_undegraded":"2024-08-17T17:42:31.137750+0000","last_fullsized":"2024-08-17T17:42:31.137750+0000","mapping_epoch":375,"log_start":"382'1900","ondisk_log_start":"382'1900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8835,"log_dups_size":0,"ondisk_log_size":8835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10735,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":196,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430280+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:42:23.430280+0000","last_peered":"2024-08-17T17:42:23.430280+0000","last_clean":"2024-08-17T17:42:23.430280+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:42:23.430280+0000","last_undegraded":"2024-08-17T17:42:23.430280+0000","last_fullsized":"2024-08-17T17:42:23.430280+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":669,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.462804+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:42:23.462804+0000","last_peered":"2024-08-17T17:42:23.462804+0000","last_clean":"2024-08-17T17:42:23.462804+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:42:23.462804+0000","last_undegraded":"2024-08-17T17:42:23.462804+0000","last_fullsized":"2024-08-17T17:42:23.462804+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"401'10676","reported_seq":10740,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.231568+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:42:29.231568+0000","last_peered":"2024-08-17T17:42:29.231568+0000","last_clean":"2024-08-17T17:42:29.231568+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:42:29.231568+0000","last_undegraded":"2024-08-17T17:42:29.231568+0000","last_fullsized":"2024-08-17T17:42:29.231568+0000","mapping_epoch":375,"log_start":"384'3600","ondisk_log_start":"384'3600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10676,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":756,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878273+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:42:07.878273+0000","last_peered":"2024-08-17T17:42:07.878273+0000","last_clean":"2024-08-17T17:42:07.878273+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:42:07.878273+0000","last_undegraded":"2024-08-17T17:42:07.878273+0000","last_fullsized":"2024-08-17T17:42:07.878273+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":684,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.451876+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:42:23.451876+0000","last_peered":"2024-08-17T17:42:23.451876+0000","last_clean":"2024-08-17T17:42:23.451876+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:42:23.451876+0000","last_undegraded":"2024-08-17T17:42:23.451876+0000","last_fullsized":"2024-08-17T17:42:23.451876+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":316,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903131+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:42:07.903131+0000","last_peered":"2024-08-17T17:42:07.903131+0000","last_clean":"2024-08-17T17:42:07.903131+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:42:07.903131+0000","last_undegraded":"2024-08-17T17:42:07.903131+0000","last_fullsized":"2024-08-17T17:42:07.903131+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"401'10236","reported_seq":10300,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:26.267117+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:42:26.267117+0000","last_peered":"2024-08-17T17:42:26.267117+0000","last_clean":"2024-08-17T17:42:26.267117+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:42:26.267117+0000","last_undegraded":"2024-08-17T17:42:26.267117+0000","last_fullsized":"2024-08-17T17:42:26.267117+0000","mapping_epoch":375,"log_start":"383'3000","ondisk_log_start":"383'3000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7236,"log_dups_size":0,"ondisk_log_size":7236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10236,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":417,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875184+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:42:07.875184+0000","last_peered":"2024-08-17T17:42:07.875184+0000","last_clean":"2024-08-17T17:42:07.875184+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:42:07.875184+0000","last_undegraded":"2024-08-17T17:42:07.875184+0000","last_fullsized":"2024-08-17T17:42:07.875184+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":322,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903110+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:42:07.903110+0000","last_peered":"2024-08-17T17:42:07.903110+0000","last_clean":"2024-08-17T17:42:07.903110+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:42:07.903110+0000","last_undegraded":"2024-08-17T17:42:07.903110+0000","last_fullsized":"2024-08-17T17:42:07.903110+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":292,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903347+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:42:07.903347+0000","last_peered":"2024-08-17T17:42:07.903347+0000","last_clean":"2024-08-17T17:42:07.903347+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:42:07.903347+0000","last_undegraded":"2024-08-17T17:42:07.903347+0000","last_fullsized":"2024-08-17T17:42:07.903347+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"401'10599","reported_seq":10635,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:30.788533+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:42:30.788533+0000","last_peered":"2024-08-17T17:42:30.788533+0000","last_clean":"2024-08-17T17:42:30.788533+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:42:30.788533+0000","last_undegraded":"2024-08-17T17:42:30.788533+0000","last_fullsized":"2024-08-17T17:42:30.788533+0000","mapping_epoch":375,"log_start":"384'3538","ondisk_log_start":"384'3538","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317521920,"num_objects":4845,"num_object_clones":0,"num_object_copies":14535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10599,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":293,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.902732+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:42:07.902732+0000","last_peered":"2024-08-17T17:42:07.902732+0000","last_clean":"2024-08-17T17:42:07.902732+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:42:07.902732+0000","last_undegraded":"2024-08-17T17:42:07.902732+0000","last_fullsized":"2024-08-17T17:42:07.902732+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903316+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:42:07.903316+0000","last_peered":"2024-08-17T17:42:07.903316+0000","last_clean":"2024-08-17T17:42:07.903316+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:42:07.903316+0000","last_undegraded":"2024-08-17T17:42:07.903316+0000","last_fullsized":"2024-08-17T17:42:07.903316+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"401'10442","reported_seq":10506,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.296080+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:42:29.296080+0000","last_peered":"2024-08-17T17:42:29.296080+0000","last_clean":"2024-08-17T17:42:29.296080+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:42:29.296080+0000","last_undegraded":"2024-08-17T17:42:29.296080+0000","last_fullsized":"2024-08-17T17:42:29.296080+0000","mapping_epoch":375,"log_start":"384'3400","ondisk_log_start":"384'3400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323354624,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10442,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":417,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875083+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:42:07.875083+0000","last_peered":"2024-08-17T17:42:07.875083+0000","last_clean":"2024-08-17T17:42:07.875083+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:42:07.875083+0000","last_undegraded":"2024-08-17T17:42:07.875083+0000","last_fullsized":"2024-08-17T17:42:07.875083+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":664,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.455732+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:42:23.455732+0000","last_peered":"2024-08-17T17:42:23.455732+0000","last_clean":"2024-08-17T17:42:23.455732+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:42:23.455732+0000","last_undegraded":"2024-08-17T17:42:23.455732+0000","last_fullsized":"2024-08-17T17:42:23.455732+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":652,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.437481+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:42:23.437481+0000","last_peered":"2024-08-17T17:42:23.437481+0000","last_clean":"2024-08-17T17:42:23.437481+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:42:23.437481+0000","last_undegraded":"2024-08-17T17:42:23.437481+0000","last_fullsized":"2024-08-17T17:42:23.437481+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"401'10849","reported_seq":10913,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.236448+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:42:29.236448+0000","last_peered":"2024-08-17T17:42:29.236448+0000","last_clean":"2024-08-17T17:42:29.236448+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:42:29.236448+0000","last_undegraded":"2024-08-17T17:42:29.236448+0000","last_fullsized":"2024-08-17T17:42:29.236448+0000","mapping_epoch":375,"log_start":"384'3800","ondisk_log_start":"384'3800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7049,"log_dups_size":0,"ondisk_log_size":7049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324468736,"num_objects":4951,"num_object_clones":0,"num_object_copies":14853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10849,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":740,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878230+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:42:07.878230+0000","last_peered":"2024-08-17T17:42:07.878230+0000","last_clean":"2024-08-17T17:42:07.878230+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:42:07.878230+0000","last_undegraded":"2024-08-17T17:42:07.878230+0000","last_fullsized":"2024-08-17T17:42:07.878230+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":994,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.874848+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:42:07.874848+0000","last_peered":"2024-08-17T17:42:07.874848+0000","last_clean":"2024-08-17T17:42:07.874848+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:42:07.874848+0000","last_undegraded":"2024-08-17T17:42:07.874848+0000","last_fullsized":"2024-08-17T17:42:07.874848+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":648,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.455963+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:42:23.455963+0000","last_peered":"2024-08-17T17:42:23.455963+0000","last_clean":"2024-08-17T17:42:23.455963+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:42:23.455963+0000","last_undegraded":"2024-08-17T17:42:23.455963+0000","last_fullsized":"2024-08-17T17:42:23.455963+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:15.596149+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:42:15.596149+0000","last_peered":"2024-08-17T17:42:15.596149+0000","last_clean":"2024-08-17T17:42:15.596149+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:42:15.596149+0000","last_undegraded":"2024-08-17T17:42:15.596149+0000","last_fullsized":"2024-08-17T17:42:15.596149+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"401'10689","reported_seq":10753,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:30.779245+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:42:30.779245+0000","last_peered":"2024-08-17T17:42:30.779245+0000","last_clean":"2024-08-17T17:42:30.779245+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:42:30.779245+0000","last_undegraded":"2024-08-17T17:42:30.779245+0000","last_fullsized":"2024-08-17T17:42:30.779245+0000","mapping_epoch":375,"log_start":"382'1500","ondisk_log_start":"382'1500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9189,"log_dups_size":0,"ondisk_log_size":9189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325648384,"num_objects":4969,"num_object_clones":0,"num_object_copies":14907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10689,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":465,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.447129+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:42:23.447129+0000","last_peered":"2024-08-17T17:42:23.447129+0000","last_clean":"2024-08-17T17:42:23.447129+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:42:23.447129+0000","last_undegraded":"2024-08-17T17:42:23.447129+0000","last_fullsized":"2024-08-17T17:42:23.447129+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":780,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878139+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:42:07.878139+0000","last_peered":"2024-08-17T17:42:07.878139+0000","last_clean":"2024-08-17T17:42:07.878139+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:42:07.878139+0000","last_undegraded":"2024-08-17T17:42:07.878139+0000","last_fullsized":"2024-08-17T17:42:07.878139+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":670,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448566+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:42:23.448566+0000","last_peered":"2024-08-17T17:42:23.448566+0000","last_clean":"2024-08-17T17:42:23.448566+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:42:23.448566+0000","last_undegraded":"2024-08-17T17:42:23.448566+0000","last_fullsized":"2024-08-17T17:42:23.448566+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":666,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448615+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:42:23.448615+0000","last_peered":"2024-08-17T17:42:23.448615+0000","last_clean":"2024-08-17T17:42:23.448615+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:42:23.448615+0000","last_undegraded":"2024-08-17T17:42:23.448615+0000","last_fullsized":"2024-08-17T17:42:23.448615+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"401'10667","reported_seq":10703,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.116512+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:42:31.116512+0000","last_peered":"2024-08-17T17:42:31.116512+0000","last_clean":"2024-08-17T17:42:31.116512+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:42:31.116512+0000","last_undegraded":"2024-08-17T17:42:31.116512+0000","last_fullsized":"2024-08-17T17:42:31.116512+0000","mapping_epoch":375,"log_start":"382'1800","ondisk_log_start":"382'1800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8867,"log_dups_size":0,"ondisk_log_size":8867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317521920,"num_objects":4845,"num_object_clones":0,"num_object_copies":14535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10667,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":465,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878055+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:42:07.878055+0000","last_peered":"2024-08-17T17:42:07.878055+0000","last_clean":"2024-08-17T17:42:07.878055+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:42:07.878055+0000","last_undegraded":"2024-08-17T17:42:07.878055+0000","last_fullsized":"2024-08-17T17:42:07.878055+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":471,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.453727+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:42:23.453727+0000","last_peered":"2024-08-17T17:42:23.453727+0000","last_clean":"2024-08-17T17:42:23.453727+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:42:23.453727+0000","last_undegraded":"2024-08-17T17:42:23.453727+0000","last_fullsized":"2024-08-17T17:42:23.453727+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":659,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.447526+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:42:23.447526+0000","last_peered":"2024-08-17T17:42:23.447526+0000","last_clean":"2024-08-17T17:42:23.447526+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:42:23.447526+0000","last_undegraded":"2024-08-17T17:42:23.447526+0000","last_fullsized":"2024-08-17T17:42:23.447526+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":706,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.442383+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:42:23.442383+0000","last_peered":"2024-08-17T17:42:23.442383+0000","last_clean":"2024-08-17T17:42:23.442383+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:42:23.442383+0000","last_undegraded":"2024-08-17T17:42:23.442383+0000","last_fullsized":"2024-08-17T17:42:23.442383+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"401'10668","reported_seq":10732,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:30.720254+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:42:30.720254+0000","last_peered":"2024-08-17T17:42:30.720254+0000","last_clean":"2024-08-17T17:42:30.720254+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:42:30.720254+0000","last_undegraded":"2024-08-17T17:42:30.720254+0000","last_fullsized":"2024-08-17T17:42:30.720254+0000","mapping_epoch":375,"log_start":"382'1500","ondisk_log_start":"382'1500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9168,"log_dups_size":0,"ondisk_log_size":9168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321126400,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10668,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":417,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.874949+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:42:07.874949+0000","last_peered":"2024-08-17T17:42:07.874949+0000","last_clean":"2024-08-17T17:42:07.874949+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:42:07.874949+0000","last_undegraded":"2024-08-17T17:42:07.874949+0000","last_fullsized":"2024-08-17T17:42:07.874949+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1117,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875266+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:42:07.875266+0000","last_peered":"2024-08-17T17:42:07.875266+0000","last_clean":"2024-08-17T17:42:07.875266+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:42:07.875266+0000","last_undegraded":"2024-08-17T17:42:07.875266+0000","last_fullsized":"2024-08-17T17:42:07.875266+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":484,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.437046+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:42:23.437046+0000","last_peered":"2024-08-17T17:42:23.437046+0000","last_clean":"2024-08-17T17:42:23.437046+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:42:23.437046+0000","last_undegraded":"2024-08-17T17:42:23.437046+0000","last_fullsized":"2024-08-17T17:42:23.437046+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.874533+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:42:07.874533+0000","last_peered":"2024-08-17T17:42:07.874533+0000","last_clean":"2024-08-17T17:42:07.874533+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:42:07.874533+0000","last_undegraded":"2024-08-17T17:42:07.874533+0000","last_fullsized":"2024-08-17T17:42:07.874533+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"401'10620","reported_seq":10656,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.076467+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:42:31.076467+0000","last_peered":"2024-08-17T17:42:31.076467+0000","last_clean":"2024-08-17T17:42:31.076467+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:42:31.076467+0000","last_undegraded":"2024-08-17T17:42:31.076467+0000","last_fullsized":"2024-08-17T17:42:31.076467+0000","mapping_epoch":375,"log_start":"382'1700","ondisk_log_start":"382'1700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8920,"log_dups_size":0,"ondisk_log_size":8920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320339968,"num_objects":4888,"num_object_clones":0,"num_object_copies":14664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10620,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":462,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.456096+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:42:23.456096+0000","last_peered":"2024-08-17T17:42:23.456096+0000","last_clean":"2024-08-17T17:42:23.456096+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:42:23.456096+0000","last_undegraded":"2024-08-17T17:42:23.456096+0000","last_fullsized":"2024-08-17T17:42:23.456096+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":186,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875620+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:42:07.875620+0000","last_peered":"2024-08-17T17:42:07.875620+0000","last_clean":"2024-08-17T17:42:07.875620+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:42:07.875620+0000","last_undegraded":"2024-08-17T17:42:07.875620+0000","last_fullsized":"2024-08-17T17:42:07.875620+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":951,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.442746+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:42:23.442746+0000","last_peered":"2024-08-17T17:42:23.442746+0000","last_clean":"2024-08-17T17:42:23.442746+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:42:23.442746+0000","last_undegraded":"2024-08-17T17:42:23.442746+0000","last_fullsized":"2024-08-17T17:42:23.442746+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":675,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.451071+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:42:23.451071+0000","last_peered":"2024-08-17T17:42:23.451071+0000","last_clean":"2024-08-17T17:42:23.451071+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:42:23.451071+0000","last_undegraded":"2024-08-17T17:42:23.451071+0000","last_fullsized":"2024-08-17T17:42:23.451071+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"401'10826","reported_seq":10862,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:30.791789+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:42:30.791789+0000","last_peered":"2024-08-17T17:42:30.791789+0000","last_clean":"2024-08-17T17:42:30.791789+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:42:30.791789+0000","last_undegraded":"2024-08-17T17:42:30.791789+0000","last_fullsized":"2024-08-17T17:42:30.791789+0000","mapping_epoch":375,"log_start":"384'3803","ondisk_log_start":"384'3803","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324403200,"num_objects":4950,"num_object_clones":0,"num_object_copies":14850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":468,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448271+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:42:23.448271+0000","last_peered":"2024-08-17T17:42:23.448271+0000","last_clean":"2024-08-17T17:42:23.448271+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:42:23.448271+0000","last_undegraded":"2024-08-17T17:42:23.448271+0000","last_fullsized":"2024-08-17T17:42:23.448271+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":672,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.442664+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:42:23.442664+0000","last_peered":"2024-08-17T17:42:23.442664+0000","last_clean":"2024-08-17T17:42:23.442664+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:42:23.442664+0000","last_undegraded":"2024-08-17T17:42:23.442664+0000","last_fullsized":"2024-08-17T17:42:23.442664+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":970,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:10.712290+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:42:10.712290+0000","last_peered":"2024-08-17T17:42:10.712290+0000","last_clean":"2024-08-17T17:42:10.712290+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:42:10.712290+0000","last_undegraded":"2024-08-17T17:42:10.712290+0000","last_fullsized":"2024-08-17T17:42:10.712290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":670,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.462610+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:42:23.462610+0000","last_peered":"2024-08-17T17:42:23.462610+0000","last_clean":"2024-08-17T17:42:23.462610+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:42:23.462610+0000","last_undegraded":"2024-08-17T17:42:23.462610+0000","last_fullsized":"2024-08-17T17:42:23.462610+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"401'10853","reported_seq":10887,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:30.739504+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:42:30.739504+0000","last_peered":"2024-08-17T17:42:30.739504+0000","last_clean":"2024-08-17T17:42:30.739504+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:42:30.739504+0000","last_undegraded":"2024-08-17T17:42:30.739504+0000","last_fullsized":"2024-08-17T17:42:30.739504+0000","mapping_epoch":375,"log_start":"384'3814","ondisk_log_start":"384'3814","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747864,"num_objects":4941,"num_object_clones":0,"num_object_copies":14823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4941,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10853,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":417,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903167+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:42:07.903167+0000","last_peered":"2024-08-17T17:42:07.903167+0000","last_clean":"2024-08-17T17:42:07.903167+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:42:07.903167+0000","last_undegraded":"2024-08-17T17:42:07.903167+0000","last_fullsized":"2024-08-17T17:42:07.903167+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":662,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.456383+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:42:23.456383+0000","last_peered":"2024-08-17T17:42:23.456383+0000","last_clean":"2024-08-17T17:42:23.456383+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:42:23.456383+0000","last_undegraded":"2024-08-17T17:42:23.456383+0000","last_fullsized":"2024-08-17T17:42:23.456383+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1182,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.445280+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:42:23.445280+0000","last_peered":"2024-08-17T17:42:23.445280+0000","last_clean":"2024-08-17T17:42:23.445280+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:42:23.445280+0000","last_undegraded":"2024-08-17T17:42:23.445280+0000","last_fullsized":"2024-08-17T17:42:23.445280+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":660,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.455492+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:42:23.455492+0000","last_peered":"2024-08-17T17:42:23.455492+0000","last_clean":"2024-08-17T17:42:23.455492+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:42:23.455492+0000","last_undegraded":"2024-08-17T17:42:23.455492+0000","last_fullsized":"2024-08-17T17:42:23.455492+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"401'10799","reported_seq":10863,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.296275+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:42:29.296275+0000","last_peered":"2024-08-17T17:42:29.296275+0000","last_clean":"2024-08-17T17:42:29.296275+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:42:29.296275+0000","last_undegraded":"2024-08-17T17:42:29.296275+0000","last_fullsized":"2024-08-17T17:42:29.296275+0000","mapping_epoch":375,"log_start":"384'3800","ondisk_log_start":"384'3800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337969152,"num_objects":5157,"num_object_clones":0,"num_object_copies":15471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10799,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":462,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.452304+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:42:23.452304+0000","last_peered":"2024-08-17T17:42:23.452304+0000","last_clean":"2024-08-17T17:42:23.452304+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:42:23.452304+0000","last_undegraded":"2024-08-17T17:42:23.452304+0000","last_fullsized":"2024-08-17T17:42:23.452304+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":681,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448370+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:42:23.448370+0000","last_peered":"2024-08-17T17:42:23.448370+0000","last_clean":"2024-08-17T17:42:23.448370+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:42:23.448370+0000","last_undegraded":"2024-08-17T17:42:23.448370+0000","last_fullsized":"2024-08-17T17:42:23.448370+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":33,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.878494+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:42:07.878494+0000","last_peered":"2024-08-17T17:42:07.878494+0000","last_clean":"2024-08-17T17:42:07.878494+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:42:07.878494+0000","last_undegraded":"2024-08-17T17:42:07.878494+0000","last_fullsized":"2024-08-17T17:42:07.878494+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":344,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.902965+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:42:07.902965+0000","last_peered":"2024-08-17T17:42:07.902965+0000","last_clean":"2024-08-17T17:42:07.902965+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:42:07.902965+0000","last_undegraded":"2024-08-17T17:42:07.902965+0000","last_fullsized":"2024-08-17T17:42:07.902965+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903124+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:42:07.903124+0000","last_peered":"2024-08-17T17:42:07.903124+0000","last_clean":"2024-08-17T17:42:07.903124+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:42:07.903124+0000","last_undegraded":"2024-08-17T17:42:07.903124+0000","last_fullsized":"2024-08-17T17:42:07.903124+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":317,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.903091+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:42:07.903091+0000","last_peered":"2024-08-17T17:42:07.903091+0000","last_clean":"2024-08-17T17:42:07.903091+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:42:07.903091+0000","last_undegraded":"2024-08-17T17:42:07.903091+0000","last_fullsized":"2024-08-17T17:42:07.903091+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":657,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.450921+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:42:23.450921+0000","last_peered":"2024-08-17T17:42:23.450921+0000","last_clean":"2024-08-17T17:42:23.450921+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:42:23.450921+0000","last_undegraded":"2024-08-17T17:42:23.450921+0000","last_fullsized":"2024-08-17T17:42:23.450921+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":402,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.883066+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:42:07.883066+0000","last_peered":"2024-08-17T17:42:07.883066+0000","last_clean":"2024-08-17T17:42:07.883066+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:42:07.883066+0000","last_undegraded":"2024-08-17T17:42:07.883066+0000","last_fullsized":"2024-08-17T17:42:07.883066+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":185,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.877137+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:42:07.877137+0000","last_peered":"2024-08-17T17:42:07.877137+0000","last_clean":"2024-08-17T17:42:07.877137+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:42:07.877137+0000","last_undegraded":"2024-08-17T17:42:07.877137+0000","last_fullsized":"2024-08-17T17:42:07.877137+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":625,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.453504+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:42:23.453504+0000","last_peered":"2024-08-17T17:42:23.453504+0000","last_clean":"2024-08-17T17:42:23.453504+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:42:23.453504+0000","last_undegraded":"2024-08-17T17:42:23.453504+0000","last_fullsized":"2024-08-17T17:42:23.453504+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":683,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448813+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:42:23.448813+0000","last_peered":"2024-08-17T17:42:23.448813+0000","last_clean":"2024-08-17T17:42:23.448813+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:42:23.448813+0000","last_undegraded":"2024-08-17T17:42:23.448813+0000","last_fullsized":"2024-08-17T17:42:23.448813+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":674,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.455409+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:42:23.455409+0000","last_peered":"2024-08-17T17:42:23.455409+0000","last_clean":"2024-08-17T17:42:23.455409+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:42:23.455409+0000","last_undegraded":"2024-08-17T17:42:23.455409+0000","last_fullsized":"2024-08-17T17:42:23.455409+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.877385+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:42:07.877385+0000","last_peered":"2024-08-17T17:42:07.877385+0000","last_clean":"2024-08-17T17:42:07.877385+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:42:07.877385+0000","last_undegraded":"2024-08-17T17:42:07.877385+0000","last_fullsized":"2024-08-17T17:42:07.877385+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":678,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.455644+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:42:23.455644+0000","last_peered":"2024-08-17T17:42:23.455644+0000","last_clean":"2024-08-17T17:42:23.455644+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:42:23.455644+0000","last_undegraded":"2024-08-17T17:42:23.455644+0000","last_fullsized":"2024-08-17T17:42:23.455644+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":656,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448540+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:42:23.448540+0000","last_peered":"2024-08-17T17:42:23.448540+0000","last_clean":"2024-08-17T17:42:23.448540+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:42:23.448540+0000","last_undegraded":"2024-08-17T17:42:23.448540+0000","last_fullsized":"2024-08-17T17:42:23.448540+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":660,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.442445+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:42:23.442445+0000","last_peered":"2024-08-17T17:42:23.442445+0000","last_clean":"2024-08-17T17:42:23.442445+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:42:23.442445+0000","last_undegraded":"2024-08-17T17:42:23.442445+0000","last_fullsized":"2024-08-17T17:42:23.442445+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":675,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.451894+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:42:23.451894+0000","last_peered":"2024-08-17T17:42:23.451894+0000","last_clean":"2024-08-17T17:42:23.451894+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:42:23.451894+0000","last_undegraded":"2024-08-17T17:42:23.451894+0000","last_fullsized":"2024-08-17T17:42:23.451894+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":292,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:20.596762+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:42:20.596762+0000","last_peered":"2024-08-17T17:42:20.596762+0000","last_clean":"2024-08-17T17:42:20.596762+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:42:20.596762+0000","last_undegraded":"2024-08-17T17:42:20.596762+0000","last_fullsized":"2024-08-17T17:42:20.596762+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":188,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.442874+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:42:23.442874+0000","last_peered":"2024-08-17T17:42:23.442874+0000","last_clean":"2024-08-17T17:42:23.442874+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:42:23.442874+0000","last_undegraded":"2024-08-17T17:42:23.442874+0000","last_fullsized":"2024-08-17T17:42:23.442874+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.462331+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:42:23.462331+0000","last_peered":"2024-08-17T17:42:23.462331+0000","last_clean":"2024-08-17T17:42:23.462331+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:42:23.462331+0000","last_undegraded":"2024-08-17T17:42:23.462331+0000","last_fullsized":"2024-08-17T17:42:23.462331+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":292,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:20.596778+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:42:20.596778+0000","last_peered":"2024-08-17T17:42:20.596778+0000","last_clean":"2024-08-17T17:42:20.596778+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:42:20.596778+0000","last_undegraded":"2024-08-17T17:42:20.596778+0000","last_fullsized":"2024-08-17T17:42:20.596778+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":189,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.874672+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:42:07.874672+0000","last_peered":"2024-08-17T17:42:07.874672+0000","last_clean":"2024-08-17T17:42:07.874672+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:42:07.874672+0000","last_undegraded":"2024-08-17T17:42:07.874672+0000","last_fullsized":"2024-08-17T17:42:07.874672+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":292,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:20.596744+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:42:20.596744+0000","last_peered":"2024-08-17T17:42:20.596744+0000","last_clean":"2024-08-17T17:42:20.596744+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:42:20.596744+0000","last_undegraded":"2024-08-17T17:42:20.596744+0000","last_fullsized":"2024-08-17T17:42:20.596744+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":199,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.875775+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:42:07.875775+0000","last_peered":"2024-08-17T17:42:07.875775+0000","last_clean":"2024-08-17T17:42:07.875775+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:42:07.875775+0000","last_undegraded":"2024-08-17T17:42:07.875775+0000","last_fullsized":"2024-08-17T17:42:07.875775+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":680,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.453813+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:42:23.453813+0000","last_peered":"2024-08-17T17:42:23.453813+0000","last_clean":"2024-08-17T17:42:23.453813+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:42:23.453813+0000","last_undegraded":"2024-08-17T17:42:23.453813+0000","last_fullsized":"2024-08-17T17:42:23.453813+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":681,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.452158+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:42:23.452158+0000","last_peered":"2024-08-17T17:42:23.452158+0000","last_clean":"2024-08-17T17:42:23.452158+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:42:23.452158+0000","last_undegraded":"2024-08-17T17:42:23.452158+0000","last_fullsized":"2024-08-17T17:42:23.452158+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":307,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.902685+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:42:07.902685+0000","last_peered":"2024-08-17T17:42:07.902685+0000","last_clean":"2024-08-17T17:42:07.902685+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:42:07.902685+0000","last_undegraded":"2024-08-17T17:42:07.902685+0000","last_fullsized":"2024-08-17T17:42:07.902685+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":701,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.448720+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:42:23.448720+0000","last_peered":"2024-08-17T17:42:23.448720+0000","last_clean":"2024-08-17T17:42:23.448720+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:42:23.448720+0000","last_undegraded":"2024-08-17T17:42:23.448720+0000","last_fullsized":"2024-08-17T17:42:23.448720+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5212667928,"num_objects":79540,"num_object_clones":0,"num_object_copies":238620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2096,"num_objects_misplaced":4990,"num_objects_unfound":0,"num_objects_dirty":79540,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170446,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":713,"num_bytes_recovered":46727168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15907172352,"data_stored":15907160136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126691,"ondisk_log_size":126691,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201748,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3345324,"kb_used_data":2552100,"kb_used_omap":1,"kb_used_meta":793214,"kb_avail":90399828,"statfs":{"total":95995035648,"available":92569423872,"internally_reserved":0,"allocated":2613350400,"data_stored":2611372400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":812251595},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922244,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1328216,"kb_used_data":969872,"kb_used_omap":7,"kb_used_meta":358328,"kb_avail":92416936,"statfs":{"total":95995035648,"available":94634942464,"internally_reserved":0,"allocated":993148928,"data_stored":991190350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":366928566},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085768,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3299168,"kb_used_data":2855192,"kb_used_omap":7,"kb_used_meta":443896,"kb_avail":90445984,"statfs":{"total":95995035648,"available":92616687616,"internally_reserved":0,"allocated":2923716608,"data_stored":2921747139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":454549945},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986658,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1683108,"kb_used_data":971932,"kb_used_omap":1,"kb_used_meta":711166,"kb_avail":92062044,"statfs":{"total":95995035648,"available":94271533056,"internally_reserved":0,"allocated":995258368,"data_stored":993288474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":728234438},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435549,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3480212,"kb_used_data":2578060,"kb_used_omap":5,"kb_used_meta":902138,"kb_avail":90264940,"statfs":{"total":95995035648,"available":92431298560,"internally_reserved":0,"allocated":2639933440,"data_stored":2637975674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":923789385},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599075,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787028,"kb_used_data":983436,"kb_used_omap":7,"kb_used_meta":803576,"kb_avail":91958124,"statfs":{"total":95995035648,"available":94165118976,"internally_reserved":0,"allocated":1007038464,"data_stored":1005080047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":822861997},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762601,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4231884,"kb_used_data":3464388,"kb_used_omap":7,"kb_used_meta":767480,"kb_avail":89513268,"statfs":{"total":95995035648,"available":91661586432,"internally_reserved":0,"allocated":3547533312,"data_stored":3545581697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":785899937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926128,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2069592,"kb_used_data":1283984,"kb_used_omap":7,"kb_used_meta":785528,"kb_avail":91675560,"statfs":{"total":95995035648,"available":93875773440,"internally_reserved":0,"allocated":1314799616,"data_stored":1312837997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":804381276},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1294925824,"data_stored":1294925824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3536195584,"data_stored":3536191512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":999948288,"data_stored":999948288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2627338240,"data_stored":2627338240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":983695360,"data_stored":983695360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2895646720,"data_stored":2895642648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":979697664,"data_stored":979697664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2589724672,"data_stored":2589720600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:34.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:34.535 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:34 smithi175 ceph-mon[101434]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2096/237123 objects degraded (0.884%); 4990/237123 objects misplaced (2.104%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:42:34.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:34.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:34.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.692 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:34.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:34.693 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:34.693 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:34.793 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:34.825 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:34.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:34 smithi064 ceph-mon[104516]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2096/237123 objects degraded (0.884%); 4990/237123 objects misplaced (2.104%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:42:34.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:34 smithi064 ceph-mon[109246]: pgmap v1249: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2096/237123 objects degraded (0.884%); 4990/237123 objects misplaced (2.104%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:42:34.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:34.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:35.061 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:35.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:35.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:35.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:35.229 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:35.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:35.366 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:35.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:42:35.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:42:35.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.531 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:35.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:35.632 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:35.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:35.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:35.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:35.906 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:42:35.906 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:35.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:35.916 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:35.916 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:35.940 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:35.958 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:42:35.958 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:35.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:42:35.992 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:36.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:42:36.028 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:36.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:36.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:42:36.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:36.073 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:36.098 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:42:36.098 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:36.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:42:36.134 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:36.172 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:42:36.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:36.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:36.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:36.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.341 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:36.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:36.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:36.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:36.571 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:36.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:36.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:36.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:36.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:36 smithi064 ceph-mon[104516]: pgmap v1250: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2096/234090 objects degraded (0.895%); 4990/234090 objects misplaced (2.132%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:36.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1002346715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:36.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:36 smithi064 ceph-mon[109246]: pgmap v1250: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2096/234090 objects degraded (0.895%); 4990/234090 objects misplaced (2.132%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:36.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1002346715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:36.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:36.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:36 smithi175 ceph-mon[101434]: pgmap v1250: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2096/234090 objects degraded (0.895%); 4990/234090 objects misplaced (2.132%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:42:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1002346715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:36.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.032 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:37.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:37.171 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:37.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:37.311 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:37.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:37.446 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:37.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:37.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:37.601 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:37.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:37.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:37.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:37.854 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:37.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:37.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:38.029 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:38.131 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:38.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:38.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:38.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:38.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:38.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:38.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:38.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:38.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:38 smithi064 ceph-mon[104516]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:38 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2281/225369 objects degraded (1.012%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:38 smithi064 ceph-mon[109246]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:38.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:38 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2281/225369 objects degraded (1.012%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.709 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:38.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:38.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:38.846 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:38 smithi175 ceph-mon[101434]: pgmap v1251: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:38 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2281/225369 objects degraded (1.012%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:38.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:38.982 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:39.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:39.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177846 9.5 9:a660b74d:::benchmark_data_smithi064_275711_object52851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.083755+0000", 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.056529376999999999, 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.056559698999999998, 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:39.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177846 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083755+0000", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083755+0000", 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083756+0000", 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8999999999999995e-07 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083760+0000", 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3429999999999998e-06 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083772+0000", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2336e-05 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.083780+0000", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1930000000000004e-06 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.105084+0000", 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021303981 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.106034+0000", 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095091399999999997 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118871+0000", 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012836924 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118989+0000", 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117576 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119269+0000", 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000280511 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134316+0000", 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01504632 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134330+0000", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4262e-05 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134370+0000", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9502999999999998e-05 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134425+0000", 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5207000000000002e-05 2024-08-17T17:42:39.148 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177884 9.9 9:9f499f38:::benchmark_data_smithi064_275711_object52889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.132975+0000", 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0073091249999999997, 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074379119999999996, 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177884 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.132975+0000", 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.132975+0000", 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.132976+0000", 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1930000000000001e-06 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.132983+0000", 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2700000000000001e-06 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.132999+0000", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6721000000000001e-05 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133007+0000", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6059999999999997e-06 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133192+0000", 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000184871 2024-08-17T17:42:39.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133709+0000", 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00051701500000000001 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133852+0000", 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143282 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177886 9.9 9:97cb6cdc:::benchmark_data_smithi064_275711_object52891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.133729+0000", 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0065547870000000003, 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067450519999999996, 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177886 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133729+0000", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133729+0000", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133730+0000", 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.029e-06 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133735+0000", 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8740000000000003e-06 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133747+0000", 2024-08-17T17:42:39.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2018000000000001e-05 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133760+0000", 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2619999999999999e-05 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138946+0000", 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051855310000000002 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.139066+0000", 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012074900000000001 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.139191+0000", 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124225 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177887 9.b 9:d98e2f0a:::benchmark_data_smithi064_275711_object52892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.133776+0000", 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0065078100000000002, 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067574009999999997, 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.159 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177887 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133776+0000", 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133776+0000", 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133777+0000", 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0699999999999997e-07 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133780+0000", 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7360000000000001e-06 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133788+0000", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9540000000000004e-06 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133793+0000", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8440000000000003e-06 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.133927+0000", 2024-08-17T17:42:39.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133287 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134045+0000", 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117982 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134159+0000", 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114689 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.139128+0000", 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049681120000000002 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.139453+0000", 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032558499999999998 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.164 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45827.0:177872 9.5 e402/393)", 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.138143+0000", 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0021409799999999998, 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002452166, 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138143+0000", 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138143+0000", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138145+0000", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.288e-06 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138149+0000", 2024-08-17T17:42:39.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138165+0000", 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5761e-05 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138173+0000", 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3299999999999999e-06 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.167 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177891 9.5 9.23fdffd5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.138197+0000", 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.002086994, 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024355470000000001, 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177891 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138197+0000", 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138197+0000", 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138199+0000", 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9829999999999999e-06 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138204+0000", 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8559999999999998e-06 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138217+0000", 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2869e-05 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138224+0000", 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4850000000000003e-06 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45827.0:177884 9.9 e402/375)", 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.138313+0000", 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0019712639999999999, 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023597129999999998, 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138313+0000", 2024-08-17T17:42:39.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138313+0000", 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138314+0000", 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1399999999999997e-07 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138317+0000", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.456e-06 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138328+0000", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1425e-05 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.138334+0000", 2024-08-17T17:42:39.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7490000000000004e-06 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177872 9.5 9:af270c80:::benchmark_data_smithi064_275711_object52877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.118681+0000", 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021603410999999999, 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022035244999999998, 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.175 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177872 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118681+0000", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118681+0000", 2024-08-17T17:42:39.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118682+0000", 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4169999999999999e-06 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118688+0000", 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.592e-06 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118698+0000", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0433e-05 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.118706+0000", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9880000000000004e-06 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.129457+0000", 2024-08-17T17:42:39.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010750381999999999 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.129631+0000", 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017482399999999999 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.129799+0000", 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016787099999999999 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.179 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:177874 9.5 9.2c3d49c5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.119152+0000", 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021132370000000001, 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021619520999999999, 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 177874 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.180 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119152+0000", 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119152+0000", 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119153+0000", 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8400000000000003e-07 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119157+0000", 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0339999999999998e-06 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119168+0000", 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0824e-05 2024-08-17T17:42:39.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.119180+0000", 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2328e-05 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45827.0:177846 9.5 e402/393 ondisk, result = 0)", 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:39.130205+0000", 2024-08-17T17:42:39.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010079384, 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010608009999999999, 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130205+0000", 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130205+0000", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130206+0000", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3e-07 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130210+0000", 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1840000000000001e-06 2024-08-17T17:42:39.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130218+0000", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7980000000000006e-06 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.130223+0000", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6690000000000003e-06 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134356+0000", 2024-08-17T17:42:39.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041332449999999998 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:39.134364+0000", 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1930000000000004e-06 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 10 2024-08-17T17:42:39.187 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:42:39.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:39.261 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:39.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:39.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:39.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:39.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:39.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:39.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:39.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:39.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:39.810 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:39.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:39.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:39.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:40.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:40.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:40.190 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:40.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:40.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:40.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:40.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:40.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:40.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:40.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:40.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:40.743 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:40.772 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:40.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[104516]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:40.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:40.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:40.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[109246]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:40.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:40.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:40.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:40.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:42:40.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:42:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:40 smithi175 ceph-mon[101434]: pgmap v1252: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2281/225369 objects degraded (1.012%); 4746/225369 objects misplaced (2.106%); 837 KiB/s, 13 objects/s recovering 2024-08-17T17:42:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:40.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:41.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:41.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:41.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:41.190 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:41.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:41.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:41.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:41.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:41.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:41.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:41.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:41.736 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:41.838 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:41.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:41.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:42.006 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:42.108 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:42.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:42.240 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:42.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:42.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:42.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:42.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:42.534 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:42.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:42.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:42.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:42.799 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:42.901 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:42 smithi175 ceph-mon[101434]: pgmap v1253: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 2508/214248 objects degraded (1.171%); 4463/214248 objects misplaced (2.083%); 735 KiB/s, 11 objects/s recovering 2024-08-17T17:42:42.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:42 smithi064 ceph-mon[104516]: pgmap v1253: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 2508/214248 objects degraded (1.171%); 4463/214248 objects misplaced (2.083%); 735 KiB/s, 11 objects/s recovering 2024-08-17T17:42:42.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:42 smithi064 ceph-mon[109246]: pgmap v1253: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 2508/214248 objects degraded (1.171%); 4463/214248 objects misplaced (2.083%); 735 KiB/s, 11 objects/s recovering 2024-08-17T17:42:43.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:43.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:43.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:43.206 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:43.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:43.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:43.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:43.479 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:43.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:43.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:43.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:43.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:43.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:43.887 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:43.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2508/214248 objects degraded (1.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:43.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2508/214248 objects degraded (1.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:43.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2508/214248 objects degraded (1.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:43.990 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:44.026 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:44.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:44.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:44.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:44.263 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:44.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:44.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:44.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:43] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-08-17T17:42:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:44.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:44.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:44.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:44.670 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:44.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:44.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:44.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:44 smithi064 ceph-mon[104516]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 2508/212607 objects degraded (1.180%); 4463/212607 objects misplaced (2.099%); 680 KiB/s, 10 objects/s recovering 2024-08-17T17:42:44.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:44 smithi064 ceph-mon[109246]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 2508/212607 objects degraded (1.180%); 4463/212607 objects misplaced (2.099%); 680 KiB/s, 10 objects/s recovering 2024-08-17T17:42:44.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:44.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:44 smithi175 ceph-mon[101434]: pgmap v1254: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 2508/212607 objects degraded (1.180%); 4463/212607 objects misplaced (2.099%); 680 KiB/s, 10 objects/s recovering 2024-08-17T17:42:44.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:45.026 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:45.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:45.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:45.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:45.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:45.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:45.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:45.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:45.569 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:45.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.711 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:45.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:45.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.860 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:45.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:45.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:45 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:45.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:45 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:45 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:45.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:46.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:46.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:46.159 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:46.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:46.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:46.438 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:46.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:46.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.582 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:46.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:46.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:46.716 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:46.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:46.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:46.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:46.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:46.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:47.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:47.135 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:47.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:47.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:46 smithi175 ceph-mon[101434]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2508/209367 objects degraded (1.198%); 4463/209367 objects misplaced (2.132%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[104516]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2508/209367 objects degraded (1.198%); 4463/209367 objects misplaced (2.132%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[109246]: pgmap v1255: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2508/209367 objects degraded (1.198%); 4463/209367 objects misplaced (2.132%); 613 KiB/s, 9 objects/s recovering 2024-08-17T17:42:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:47.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:47.279 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:47.284 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:47.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:47.430 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:47.531 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.579 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:47.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:47.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:47.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:47.816 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.864 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:47.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:47.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:48.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:48.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:48.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:48.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:48.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:48.392 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.454 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:48.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:48.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:48.587 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:48.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:48.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:48.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:48.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:48.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:48.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:49.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:49.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:49.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:49.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:49.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:48 smithi175 ceph-mon[101434]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 911 KiB/s, 14 objects/s recovering 2024-08-17T17:42:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2737/200103 objects degraded (1.368%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:49.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:48 smithi064 ceph-mon[104516]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 911 KiB/s, 14 objects/s recovering 2024-08-17T17:42:49.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2737/200103 objects degraded (1.368%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:49.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:48 smithi064 ceph-mon[109246]: pgmap v1256: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 911 KiB/s, 14 objects/s recovering 2024-08-17T17:42:49.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2737/200103 objects degraded (1.368%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:49.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:49.273 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:49.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:49.396 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:49.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:49.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:49.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:49.677 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:49.778 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:49.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:49.914 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:49.950 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:50.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:50.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.088 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:50.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:50.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:50.228 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:50.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:50.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:50.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:50.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:50.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:50.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:50.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:50.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:50.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:50.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:50.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:51.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:51.074 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:51.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:50 smithi175 ceph-mon[101434]: pgmap v1257: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 597 KiB/s, 9 objects/s recovering 2024-08-17T17:42:51.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:51.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:50 smithi064 ceph-mon[104516]: pgmap v1257: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 597 KiB/s, 9 objects/s recovering 2024-08-17T17:42:51.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:50 smithi064 ceph-mon[109246]: pgmap v1257: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2737/200103 objects degraded (1.368%); 4178/200103 objects misplaced (2.088%); 597 KiB/s, 9 objects/s recovering 2024-08-17T17:42:51.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:51.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:51.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:51.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:51.493 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:51.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:51.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:51.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:51.761 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:51.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:51.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:52.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:52.042 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:52.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:52.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:52.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:52.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:52.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:52.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:52.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:52.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:52.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.591 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:52.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:52.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:52.726 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:52.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.854 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:52.929 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[104516]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 2927/189156 objects degraded (1.547%); 3923/189156 objects misplaced (2.074%); 944 KiB/s, 14 objects/s recovering 2024-08-17T17:42:52.929 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:52.929 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:52.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[109246]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 2927/189156 objects degraded (1.547%); 3923/189156 objects misplaced (2.074%); 944 KiB/s, 14 objects/s recovering 2024-08-17T17:42:52.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:52.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:52.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.983 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:52.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:53.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:53.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:53.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:52 smithi175 ceph-mon[101434]: pgmap v1258: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 2927/189156 objects degraded (1.547%); 3923/189156 objects misplaced (2.074%); 944 KiB/s, 14 objects/s recovering 2024-08-17T17:42:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:42:53.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:53.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:53.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:53.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:53.507 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:53.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:53.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:53.668 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:53.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:53.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:53.811 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:42:53.812 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:42:53.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.935 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:53.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:54.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:54.071 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 2927/189156 objects degraded (1.547%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:54.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:54.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 2927/189156 objects degraded (1.547%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:42:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:42:53] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-08-17T17:42:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 2927/189156 objects degraded (1.547%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:54.207 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:54.309 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:54.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:54.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:54.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:54.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:54.616 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:54.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:54.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:54.748 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:54.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:54.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:54.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:54.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:55.021 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:55.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:55.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:55.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:54 smithi175 ceph-mon[101434]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 2927/187665 objects degraded (1.560%); 3923/187665 objects misplaced (2.090%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:54 smithi175 ceph-mon[101434]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:55.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:54 smithi064 ceph-mon[104516]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 2927/187665 objects degraded (1.560%); 3923/187665 objects misplaced (2.090%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:55.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:54 smithi064 ceph-mon[104516]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:55.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:54 smithi064 ceph-mon[109246]: pgmap v1259: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 2927/187665 objects degraded (1.560%); 3923/187665 objects misplaced (2.090%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:55.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:54 smithi064 ceph-mon[109246]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:55.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:55.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:55.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:55.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:55.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:55.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:55.571 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:55.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:55.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:55.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:55.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:55.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:55.929 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:42:55.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:55.975 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:56.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:56.101 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:56.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/653537636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[104516]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 2927/184578 objects degraded (1.586%); 3923/184578 objects misplaced (2.125%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/56206538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/653537636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[109246]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 2927/184578 objects degraded (1.586%); 3923/184578 objects misplaced (2.125%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:56.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/56206538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.203 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:56.235 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:56.337 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:56.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:56.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/653537636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:55 smithi175 ceph-mon[101434]: pgmap v1260: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 2927/184578 objects degraded (1.586%); 3923/184578 objects misplaced (2.125%); 645 KiB/s, 10 objects/s recovering 2024-08-17T17:42:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/56206538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:56.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:56.604 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:56.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:56.738 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:56.775 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:56.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:56.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.903 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:56.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:57.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:57.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:57.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:57.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:57.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:57.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:57.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[104516]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:42:57.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:57.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:57.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:56 smithi064 ceph-mon[109246]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:42:57.178 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:57.187 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1259,"stamp":"2024-08-17T17:42:53.300768+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4102494205,"num_objects":62555,"num_object_clones":0,"num_object_copies":187665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2927,"num_objects_misplaced":3923,"num_objects_unfound":0,"num_objects_dirty":62555,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":188367,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":70588643,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129276,"ondisk_log_size":129276,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18062964,"kb_used_data":12369844,"kb_used_omap":46,"kb_used_meta":5693009,"kb_avail":731898252,"statfs":{"total":767960285184,"available":749463810048,"internally_reserved":0,"allocated":12666720256,"data_stored":12650983602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5829641459},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,1,4,7,2,1,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-580714496,"num_objects":-8861,"num_object_clones":0,"num_object_copies":-26583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":419,"num_objects_misplaced":-540,"num_objects_unfound":0,"num_objects_dirty":-8861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8861,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005812"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":665,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477309+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:42:32.477309+0000","last_peered":"2024-08-17T17:42:32.477309+0000","last_clean":"2024-08-17T17:42:32.477309+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:42:32.477309+0000","last_undegraded":"2024-08-17T17:42:32.477309+0000","last_fullsized":"2024-08-17T17:42:32.477309+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":402,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425730+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:42:23.425730+0000","last_peered":"2024-08-17T17:42:23.425730+0000","last_clean":"2024-08-17T17:42:23.425730+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:42:23.425730+0000","last_undegraded":"2024-08-17T17:42:23.425730+0000","last_fullsized":"2024-08-17T17:42:23.425730+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":186,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430467+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:42:23.430467+0000","last_peered":"2024-08-17T17:42:23.430467+0000","last_clean":"2024-08-17T17:42:23.430467+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:42:23.430467+0000","last_undegraded":"2024-08-17T17:42:23.430467+0000","last_fullsized":"2024-08-17T17:42:23.430467+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":656,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486445+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:42:32.486445+0000","last_peered":"2024-08-17T17:42:32.486445+0000","last_clean":"2024-08-17T17:42:32.486445+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:42:32.486445+0000","last_undegraded":"2024-08-17T17:42:32.486445+0000","last_fullsized":"2024-08-17T17:42:32.486445+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":656,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486881+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:42:32.486881+0000","last_peered":"2024-08-17T17:42:32.486881+0000","last_clean":"2024-08-17T17:42:32.486881+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:42:32.486881+0000","last_undegraded":"2024-08-17T17:42:32.486881+0000","last_fullsized":"2024-08-17T17:42:32.486881+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":388,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425084+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:42:23.425084+0000","last_peered":"2024-08-17T17:42:23.425084+0000","last_clean":"2024-08-17T17:42:23.425084+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:42:23.425084+0000","last_undegraded":"2024-08-17T17:42:23.425084+0000","last_fullsized":"2024-08-17T17:42:23.425084+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":664,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.476842+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:42:32.476842+0000","last_peered":"2024-08-17T17:42:32.476842+0000","last_clean":"2024-08-17T17:42:32.476842+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:42:32.476842+0000","last_undegraded":"2024-08-17T17:42:32.476842+0000","last_fullsized":"2024-08-17T17:42:32.476842+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":336,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.446359+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:42:23.446359+0000","last_peered":"2024-08-17T17:42:23.446359+0000","last_clean":"2024-08-17T17:42:23.446359+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:42:23.446359+0000","last_undegraded":"2024-08-17T17:42:23.446359+0000","last_fullsized":"2024-08-17T17:42:23.446359+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"402'11698","reported_seq":13664,"reported_epoch":402,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:50.912689+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:42:50.912689+0000","last_peered":"2024-08-17T17:42:50.912689+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:42:50.912689+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"383'2600","ondisk_log_start":"383'2600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2927,"num_objects_misplaced":3923,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11698,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":949,"num_bytes_recovered":62193664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":3923},{"shards":"0,5","objects":-2904}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":398,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425590+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:42:23.425590+0000","last_peered":"2024-08-17T17:42:23.425590+0000","last_clean":"2024-08-17T17:42:23.425590+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:42:23.425590+0000","last_undegraded":"2024-08-17T17:42:23.425590+0000","last_fullsized":"2024-08-17T17:42:23.425590+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430021+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:42:23.430021+0000","last_peered":"2024-08-17T17:42:23.430021+0000","last_clean":"2024-08-17T17:42:23.430021+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:42:23.430021+0000","last_undegraded":"2024-08-17T17:42:23.430021+0000","last_fullsized":"2024-08-17T17:42:23.430021+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"402'11695","reported_seq":11761,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.321829+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:42:51.321829+0000","last_peered":"2024-08-17T17:42:51.321829+0000","last_clean":"2024-08-17T17:42:51.321829+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:42:51.321829+0000","last_undegraded":"2024-08-17T17:42:51.321829+0000","last_fullsized":"2024-08-17T17:42:51.321829+0000","mapping_epoch":375,"log_start":"385'4500","ondisk_log_start":"385'4500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258539520,"num_objects":3945,"num_object_clones":0,"num_object_copies":11835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11695,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":399,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425508+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:42:23.425508+0000","last_peered":"2024-08-17T17:42:23.425508+0000","last_clean":"2024-08-17T17:42:23.425508+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:42:23.425508+0000","last_undegraded":"2024-08-17T17:42:23.425508+0000","last_fullsized":"2024-08-17T17:42:23.425508+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":294,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436268+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:42:23.436268+0000","last_peered":"2024-08-17T17:42:23.436268+0000","last_clean":"2024-08-17T17:42:23.436268+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:42:23.436268+0000","last_undegraded":"2024-08-17T17:42:23.436268+0000","last_fullsized":"2024-08-17T17:42:23.436268+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"402'11753","reported_seq":11790,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.135019+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:42:51.135019+0000","last_peered":"2024-08-17T17:42:51.135019+0000","last_clean":"2024-08-17T17:42:51.135019+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:42:51.135019+0000","last_undegraded":"2024-08-17T17:42:51.135019+0000","last_fullsized":"2024-08-17T17:42:51.135019+0000","mapping_epoch":375,"log_start":"383'2900","ondisk_log_start":"383'2900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8853,"log_dups_size":0,"ondisk_log_size":8853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254476288,"num_objects":3883,"num_object_clones":0,"num_object_copies":11649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11753,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":416,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.427452+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:42:23.427452+0000","last_peered":"2024-08-17T17:42:23.427452+0000","last_clean":"2024-08-17T17:42:23.427452+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:42:23.427452+0000","last_undegraded":"2024-08-17T17:42:23.427452+0000","last_fullsized":"2024-08-17T17:42:23.427452+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":317,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:24.628058+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:42:24.628058+0000","last_peered":"2024-08-17T17:42:24.628058+0000","last_clean":"2024-08-17T17:42:24.628058+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:42:24.628058+0000","last_undegraded":"2024-08-17T17:42:24.628058+0000","last_fullsized":"2024-08-17T17:42:24.628058+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"402'11782","reported_seq":11819,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.106065+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:42:51.106065+0000","last_peered":"2024-08-17T17:42:51.106065+0000","last_clean":"2024-08-17T17:42:51.106065+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:42:51.106065+0000","last_undegraded":"2024-08-17T17:42:51.106065+0000","last_fullsized":"2024-08-17T17:42:51.106065+0000","mapping_epoch":375,"log_start":"383'2900","ondisk_log_start":"383'2900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254803968,"num_objects":3888,"num_object_clones":0,"num_object_copies":11664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11782,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":197,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.141505+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:42:31.141505+0000","last_peered":"2024-08-17T17:42:31.141505+0000","last_clean":"2024-08-17T17:42:31.141505+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:42:31.141505+0000","last_undegraded":"2024-08-17T17:42:31.141505+0000","last_fullsized":"2024-08-17T17:42:31.141505+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":671,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486725+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:42:32.486725+0000","last_peered":"2024-08-17T17:42:32.486725+0000","last_clean":"2024-08-17T17:42:32.486725+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:42:32.486725+0000","last_undegraded":"2024-08-17T17:42:32.486725+0000","last_fullsized":"2024-08-17T17:42:32.486725+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"402'11737","reported_seq":11803,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:49.313210+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:42:49.313210+0000","last_peered":"2024-08-17T17:42:49.313210+0000","last_clean":"2024-08-17T17:42:49.313210+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:42:49.313210+0000","last_undegraded":"2024-08-17T17:42:49.313210+0000","last_fullsized":"2024-08-17T17:42:49.313210+0000","mapping_epoch":375,"log_start":"385'4700","ondisk_log_start":"385'4700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264044544,"num_objects":4029,"num_object_clones":0,"num_object_copies":12087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11737,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":757,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.424972+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:42:23.424972+0000","last_peered":"2024-08-17T17:42:23.424972+0000","last_clean":"2024-08-17T17:42:23.424972+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:42:23.424972+0000","last_undegraded":"2024-08-17T17:42:23.424972+0000","last_fullsized":"2024-08-17T17:42:23.424972+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":686,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.480434+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:42:32.480434+0000","last_peered":"2024-08-17T17:42:32.480434+0000","last_clean":"2024-08-17T17:42:32.480434+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:42:32.480434+0000","last_undegraded":"2024-08-17T17:42:32.480434+0000","last_fullsized":"2024-08-17T17:42:32.480434+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":318,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.471863+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:42:32.471863+0000","last_peered":"2024-08-17T17:42:32.471863+0000","last_clean":"2024-08-17T17:42:32.471863+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:42:32.471863+0000","last_undegraded":"2024-08-17T17:42:32.471863+0000","last_fullsized":"2024-08-17T17:42:32.471863+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"402'11479","reported_seq":11545,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.379207+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:42:51.379207+0000","last_peered":"2024-08-17T17:42:51.379207+0000","last_clean":"2024-08-17T17:42:51.379207+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:42:51.379207+0000","last_undegraded":"2024-08-17T17:42:51.379207+0000","last_fullsized":"2024-08-17T17:42:51.379207+0000","mapping_epoch":375,"log_start":"385'4300","ondisk_log_start":"385'4300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7179,"log_dups_size":0,"ondisk_log_size":7179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11479,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":418,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430177+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:42:23.430177+0000","last_peered":"2024-08-17T17:42:23.430177+0000","last_clean":"2024-08-17T17:42:23.430177+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:42:23.430177+0000","last_undegraded":"2024-08-17T17:42:23.430177+0000","last_fullsized":"2024-08-17T17:42:23.430177+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":324,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.471647+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:42:32.471647+0000","last_peered":"2024-08-17T17:42:32.471647+0000","last_clean":"2024-08-17T17:42:32.471647+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:42:32.471647+0000","last_undegraded":"2024-08-17T17:42:32.471647+0000","last_fullsized":"2024-08-17T17:42:32.471647+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":294,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.472126+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:42:32.472126+0000","last_peered":"2024-08-17T17:42:32.472126+0000","last_clean":"2024-08-17T17:42:32.472126+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:42:32.472126+0000","last_undegraded":"2024-08-17T17:42:32.472126+0000","last_fullsized":"2024-08-17T17:42:32.472126+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"402'11652","reported_seq":11689,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:50.816146+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:42:50.816146+0000","last_peered":"2024-08-17T17:42:50.816146+0000","last_clean":"2024-08-17T17:42:50.816146+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:42:50.816146+0000","last_undegraded":"2024-08-17T17:42:50.816146+0000","last_fullsized":"2024-08-17T17:42:50.816146+0000","mapping_epoch":375,"log_start":"385'4638","ondisk_log_start":"385'4638","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248512512,"num_objects":3792,"num_object_clones":0,"num_object_copies":11376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11652,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":294,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436163+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:42:23.436163+0000","last_peered":"2024-08-17T17:42:23.436163+0000","last_clean":"2024-08-17T17:42:23.436163+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:42:23.436163+0000","last_undegraded":"2024-08-17T17:42:23.436163+0000","last_fullsized":"2024-08-17T17:42:23.436163+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":309,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.472365+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:42:32.472365+0000","last_peered":"2024-08-17T17:42:32.472365+0000","last_clean":"2024-08-17T17:42:32.472365+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:42:32.472365+0000","last_undegraded":"2024-08-17T17:42:32.472365+0000","last_fullsized":"2024-08-17T17:42:32.472365+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"402'11456","reported_seq":11522,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:49.281599+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:42:49.281599+0000","last_peered":"2024-08-17T17:42:49.281599+0000","last_clean":"2024-08-17T17:42:49.281599+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:42:49.281599+0000","last_undegraded":"2024-08-17T17:42:49.281599+0000","last_fullsized":"2024-08-17T17:42:49.281599+0000","mapping_epoch":375,"log_start":"385'4400","ondisk_log_start":"385'4400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901120,"num_objects":3920,"num_object_clones":0,"num_object_copies":11760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11456,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":418,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.428750+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:42:23.428750+0000","last_peered":"2024-08-17T17:42:23.428750+0000","last_clean":"2024-08-17T17:42:23.428750+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:42:23.428750+0000","last_undegraded":"2024-08-17T17:42:23.428750+0000","last_fullsized":"2024-08-17T17:42:23.428750+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":667,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.476094+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:42:32.476094+0000","last_peered":"2024-08-17T17:42:32.476094+0000","last_clean":"2024-08-17T17:42:32.476094+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:42:32.476094+0000","last_undegraded":"2024-08-17T17:42:32.476094+0000","last_fullsized":"2024-08-17T17:42:32.476094+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":654,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.482579+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:42:32.482579+0000","last_peered":"2024-08-17T17:42:32.482579+0000","last_clean":"2024-08-17T17:42:32.482579+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:42:32.482579+0000","last_undegraded":"2024-08-17T17:42:32.482579+0000","last_fullsized":"2024-08-17T17:42:32.482579+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"402'11876","reported_seq":11942,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:49.160935+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:42:49.160935+0000","last_peered":"2024-08-17T17:42:49.160935+0000","last_clean":"2024-08-17T17:42:49.160935+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:42:49.160935+0000","last_undegraded":"2024-08-17T17:42:49.160935+0000","last_fullsized":"2024-08-17T17:42:49.160935+0000","mapping_epoch":375,"log_start":"385'4800","ondisk_log_start":"385'4800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11876,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":741,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.424977+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:42:23.424977+0000","last_peered":"2024-08-17T17:42:23.424977+0000","last_clean":"2024-08-17T17:42:23.424977+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:42:23.424977+0000","last_undegraded":"2024-08-17T17:42:23.424977+0000","last_fullsized":"2024-08-17T17:42:23.424977+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":995,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.431774+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:42:23.431774+0000","last_peered":"2024-08-17T17:42:23.431774+0000","last_clean":"2024-08-17T17:42:23.431774+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:42:23.431774+0000","last_undegraded":"2024-08-17T17:42:23.431774+0000","last_fullsized":"2024-08-17T17:42:23.431774+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":650,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.482538+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:42:32.482538+0000","last_peered":"2024-08-17T17:42:32.482538+0000","last_clean":"2024-08-17T17:42:32.482538+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:42:32.482538+0000","last_undegraded":"2024-08-17T17:42:32.482538+0000","last_fullsized":"2024-08-17T17:42:32.482538+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":308,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.437189+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:42:23.437189+0000","last_peered":"2024-08-17T17:42:23.437189+0000","last_clean":"2024-08-17T17:42:23.437189+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:42:23.437189+0000","last_undegraded":"2024-08-17T17:42:23.437189+0000","last_fullsized":"2024-08-17T17:42:23.437189+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"402'11746","reported_seq":11812,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:50.872981+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:42:50.872981+0000","last_peered":"2024-08-17T17:42:50.872981+0000","last_clean":"2024-08-17T17:42:50.872981+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:42:50.872981+0000","last_undegraded":"2024-08-17T17:42:50.872981+0000","last_fullsized":"2024-08-17T17:42:50.872981+0000","mapping_epoch":375,"log_start":"383'2600","ondisk_log_start":"383'2600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9146,"log_dups_size":0,"ondisk_log_size":9146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256376832,"num_objects":3912,"num_object_clones":0,"num_object_copies":11736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11746,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":467,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.479749+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:42:32.479749+0000","last_peered":"2024-08-17T17:42:32.479749+0000","last_clean":"2024-08-17T17:42:32.479749+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:42:32.479749+0000","last_undegraded":"2024-08-17T17:42:32.479749+0000","last_fullsized":"2024-08-17T17:42:32.479749+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":782,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.470093+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:42:32.470093+0000","last_peered":"2024-08-17T17:42:32.470093+0000","last_clean":"2024-08-17T17:42:32.470093+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:42:32.470093+0000","last_undegraded":"2024-08-17T17:42:32.470093+0000","last_fullsized":"2024-08-17T17:42:32.470093+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":672,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.487903+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:42:32.487903+0000","last_peered":"2024-08-17T17:42:32.487903+0000","last_clean":"2024-08-17T17:42:32.487903+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:42:32.487903+0000","last_undegraded":"2024-08-17T17:42:32.487903+0000","last_fullsized":"2024-08-17T17:42:32.487903+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":668,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.487084+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:42:32.487084+0000","last_peered":"2024-08-17T17:42:32.487084+0000","last_clean":"2024-08-17T17:42:32.487084+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:42:32.487084+0000","last_undegraded":"2024-08-17T17:42:32.487084+0000","last_fullsized":"2024-08-17T17:42:32.487084+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"402'11661","reported_seq":11698,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.162289+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:42:51.162289+0000","last_peered":"2024-08-17T17:42:51.162289+0000","last_clean":"2024-08-17T17:42:51.162289+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:42:51.162289+0000","last_undegraded":"2024-08-17T17:42:51.162289+0000","last_fullsized":"2024-08-17T17:42:51.162289+0000","mapping_epoch":375,"log_start":"383'2800","ondisk_log_start":"383'2800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252379136,"num_objects":3851,"num_object_clones":0,"num_object_copies":11553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11661,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":466,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425391+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:42:23.425391+0000","last_peered":"2024-08-17T17:42:23.425391+0000","last_clean":"2024-08-17T17:42:23.425391+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:42:23.425391+0000","last_undegraded":"2024-08-17T17:42:23.425391+0000","last_fullsized":"2024-08-17T17:42:23.425391+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":473,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477553+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:42:32.477553+0000","last_peered":"2024-08-17T17:42:32.477553+0000","last_clean":"2024-08-17T17:42:32.477553+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:42:32.477553+0000","last_undegraded":"2024-08-17T17:42:32.477553+0000","last_fullsized":"2024-08-17T17:42:32.477553+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":661,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.479285+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:42:32.479285+0000","last_peered":"2024-08-17T17:42:32.479285+0000","last_clean":"2024-08-17T17:42:32.479285+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:42:32.479285+0000","last_undegraded":"2024-08-17T17:42:32.479285+0000","last_fullsized":"2024-08-17T17:42:32.479285+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":708,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.476627+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:42:32.476627+0000","last_peered":"2024-08-17T17:42:32.476627+0000","last_clean":"2024-08-17T17:42:32.476627+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:42:32.476627+0000","last_undegraded":"2024-08-17T17:42:32.476627+0000","last_fullsized":"2024-08-17T17:42:32.476627+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"402'11675","reported_seq":11741,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:50.899712+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:42:50.899712+0000","last_peered":"2024-08-17T17:42:50.899712+0000","last_clean":"2024-08-17T17:42:50.899712+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:42:50.899712+0000","last_undegraded":"2024-08-17T17:42:50.899712+0000","last_fullsized":"2024-08-17T17:42:50.899712+0000","mapping_epoch":375,"log_start":"383'2500","ondisk_log_start":"383'2500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9175,"log_dups_size":0,"ondisk_log_size":9175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255131648,"num_objects":3893,"num_object_clones":0,"num_object_copies":11679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11675,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":418,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.428641+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:42:23.428641+0000","last_peered":"2024-08-17T17:42:23.428641+0000","last_clean":"2024-08-17T17:42:23.428641+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:42:23.428641+0000","last_undegraded":"2024-08-17T17:42:23.428641+0000","last_fullsized":"2024-08-17T17:42:23.428641+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1118,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.428900+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:42:23.428900+0000","last_peered":"2024-08-17T17:42:23.428900+0000","last_clean":"2024-08-17T17:42:23.428900+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:42:23.428900+0000","last_undegraded":"2024-08-17T17:42:23.428900+0000","last_fullsized":"2024-08-17T17:42:23.428900+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":486,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.481860+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:42:32.481860+0000","last_peered":"2024-08-17T17:42:32.481860+0000","last_clean":"2024-08-17T17:42:32.481860+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:42:32.481860+0000","last_undegraded":"2024-08-17T17:42:32.481860+0000","last_fullsized":"2024-08-17T17:42:32.481860+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":49,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.428640+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:42:23.428640+0000","last_peered":"2024-08-17T17:42:23.428640+0000","last_clean":"2024-08-17T17:42:23.428640+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:42:23.428640+0000","last_undegraded":"2024-08-17T17:42:23.428640+0000","last_fullsized":"2024-08-17T17:42:23.428640+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"402'11608","reported_seq":11645,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:51.141631+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:42:51.141631+0000","last_peered":"2024-08-17T17:42:51.141631+0000","last_clean":"2024-08-17T17:42:51.141631+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:42:51.141631+0000","last_undegraded":"2024-08-17T17:42:51.141631+0000","last_fullsized":"2024-08-17T17:42:51.141631+0000","mapping_epoch":375,"log_start":"383'2700","ondisk_log_start":"383'2700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8908,"log_dups_size":0,"ondisk_log_size":8908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255590400,"num_objects":3900,"num_object_clones":0,"num_object_copies":11700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11608,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":464,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.476742+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:42:32.476742+0000","last_peered":"2024-08-17T17:42:32.476742+0000","last_clean":"2024-08-17T17:42:32.476742+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:42:32.476742+0000","last_undegraded":"2024-08-17T17:42:32.476742+0000","last_fullsized":"2024-08-17T17:42:32.476742+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430347+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:42:23.430347+0000","last_peered":"2024-08-17T17:42:23.430347+0000","last_clean":"2024-08-17T17:42:23.430347+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:42:23.430347+0000","last_undegraded":"2024-08-17T17:42:23.430347+0000","last_fullsized":"2024-08-17T17:42:23.430347+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":954,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.474175+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:42:32.474175+0000","last_peered":"2024-08-17T17:42:32.474175+0000","last_clean":"2024-08-17T17:42:32.474175+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:42:32.474175+0000","last_undegraded":"2024-08-17T17:42:32.474175+0000","last_fullsized":"2024-08-17T17:42:32.474175+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":678,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.483765+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:42:32.483765+0000","last_peered":"2024-08-17T17:42:32.483765+0000","last_clean":"2024-08-17T17:42:32.483765+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:42:32.483765+0000","last_undegraded":"2024-08-17T17:42:32.483765+0000","last_fullsized":"2024-08-17T17:42:32.483765+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"402'11876","reported_seq":11913,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:50.745443+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:42:50.745443+0000","last_peered":"2024-08-17T17:42:50.745443+0000","last_clean":"2024-08-17T17:42:50.745443+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:42:50.745443+0000","last_undegraded":"2024-08-17T17:42:50.745443+0000","last_fullsized":"2024-08-17T17:42:50.745443+0000","mapping_epoch":375,"log_start":"385'4803","ondisk_log_start":"385'4803","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255590400,"num_objects":3900,"num_object_clones":0,"num_object_copies":11700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11876,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":470,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486688+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:42:32.486688+0000","last_peered":"2024-08-17T17:42:32.486688+0000","last_clean":"2024-08-17T17:42:32.486688+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:42:32.486688+0000","last_undegraded":"2024-08-17T17:42:32.486688+0000","last_fullsized":"2024-08-17T17:42:32.486688+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":675,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.474092+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:42:32.474092+0000","last_peered":"2024-08-17T17:42:32.474092+0000","last_clean":"2024-08-17T17:42:32.474092+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:42:32.474092+0000","last_undegraded":"2024-08-17T17:42:32.474092+0000","last_fullsized":"2024-08-17T17:42:32.474092+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":971,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.427446+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:42:23.427446+0000","last_peered":"2024-08-17T17:42:23.427446+0000","last_clean":"2024-08-17T17:42:23.427446+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:42:23.427446+0000","last_undegraded":"2024-08-17T17:42:23.427446+0000","last_fullsized":"2024-08-17T17:42:23.427446+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":672,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486558+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:42:32.486558+0000","last_peered":"2024-08-17T17:42:32.486558+0000","last_clean":"2024-08-17T17:42:32.486558+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:42:32.486558+0000","last_undegraded":"2024-08-17T17:42:32.486558+0000","last_fullsized":"2024-08-17T17:42:32.486558+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"402'11894","reported_seq":11932,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:50.832253+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:42:50.832253+0000","last_peered":"2024-08-17T17:42:50.832253+0000","last_clean":"2024-08-17T17:42:50.832253+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:42:50.832253+0000","last_undegraded":"2024-08-17T17:42:50.832253+0000","last_fullsized":"2024-08-17T17:42:50.832253+0000","mapping_epoch":375,"log_start":"385'4914","ondisk_log_start":"385'4914","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524888,"num_objects":3900,"num_object_clones":0,"num_object_copies":11700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3900,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11894,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":418,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.437060+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:42:23.437060+0000","last_peered":"2024-08-17T17:42:23.437060+0000","last_clean":"2024-08-17T17:42:23.437060+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:42:23.437060+0000","last_undegraded":"2024-08-17T17:42:23.437060+0000","last_fullsized":"2024-08-17T17:42:23.437060+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":665,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477105+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:42:32.477105+0000","last_peered":"2024-08-17T17:42:32.477105+0000","last_clean":"2024-08-17T17:42:32.477105+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:42:32.477105+0000","last_undegraded":"2024-08-17T17:42:32.477105+0000","last_fullsized":"2024-08-17T17:42:32.477105+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1184,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477554+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:42:32.477554+0000","last_peered":"2024-08-17T17:42:32.477554+0000","last_clean":"2024-08-17T17:42:32.477554+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:42:32.477554+0000","last_undegraded":"2024-08-17T17:42:32.477554+0000","last_fullsized":"2024-08-17T17:42:32.477554+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":662,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.475783+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:42:32.475783+0000","last_peered":"2024-08-17T17:42:32.475783+0000","last_clean":"2024-08-17T17:42:32.475783+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:42:32.475783+0000","last_undegraded":"2024-08-17T17:42:32.475783+0000","last_fullsized":"2024-08-17T17:42:32.475783+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"402'11871","reported_seq":11937,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:49.273909+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:42:49.273909+0000","last_peered":"2024-08-17T17:42:49.273909+0000","last_clean":"2024-08-17T17:42:49.273909+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:42:49.273909+0000","last_undegraded":"2024-08-17T17:42:49.273909+0000","last_fullsized":"2024-08-17T17:42:49.273909+0000","mapping_epoch":375,"log_start":"385'4800","ondisk_log_start":"385'4800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267714560,"num_objects":4085,"num_object_clones":0,"num_object_copies":12255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11871,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":464,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477083+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:42:32.477083+0000","last_peered":"2024-08-17T17:42:32.477083+0000","last_clean":"2024-08-17T17:42:32.477083+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:42:32.477083+0000","last_undegraded":"2024-08-17T17:42:32.477083+0000","last_fullsized":"2024-08-17T17:42:32.477083+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":683,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486785+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:42:32.486785+0000","last_peered":"2024-08-17T17:42:32.486785+0000","last_clean":"2024-08-17T17:42:32.486785+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:42:32.486785+0000","last_undegraded":"2024-08-17T17:42:32.486785+0000","last_fullsized":"2024-08-17T17:42:32.486785+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":34,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.425301+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:42:23.425301+0000","last_peered":"2024-08-17T17:42:23.425301+0000","last_clean":"2024-08-17T17:42:23.425301+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:42:23.425301+0000","last_undegraded":"2024-08-17T17:42:23.425301+0000","last_fullsized":"2024-08-17T17:42:23.425301+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":345,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436372+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:42:23.436372+0000","last_peered":"2024-08-17T17:42:23.436372+0000","last_clean":"2024-08-17T17:42:23.436372+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:42:23.436372+0000","last_undegraded":"2024-08-17T17:42:23.436372+0000","last_fullsized":"2024-08-17T17:42:23.436372+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":308,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436961+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:42:23.436961+0000","last_peered":"2024-08-17T17:42:23.436961+0000","last_clean":"2024-08-17T17:42:23.436961+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:42:23.436961+0000","last_undegraded":"2024-08-17T17:42:23.436961+0000","last_fullsized":"2024-08-17T17:42:23.436961+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":318,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.437122+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:42:23.437122+0000","last_peered":"2024-08-17T17:42:23.437122+0000","last_clean":"2024-08-17T17:42:23.437122+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:42:23.437122+0000","last_undegraded":"2024-08-17T17:42:23.437122+0000","last_fullsized":"2024-08-17T17:42:23.437122+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":659,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.480638+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:42:32.480638+0000","last_peered":"2024-08-17T17:42:32.480638+0000","last_clean":"2024-08-17T17:42:32.480638+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:42:32.480638+0000","last_undegraded":"2024-08-17T17:42:32.480638+0000","last_fullsized":"2024-08-17T17:42:32.480638+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":403,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.427372+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:42:23.427372+0000","last_peered":"2024-08-17T17:42:23.427372+0000","last_clean":"2024-08-17T17:42:23.427372+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:42:23.427372+0000","last_undegraded":"2024-08-17T17:42:23.427372+0000","last_fullsized":"2024-08-17T17:42:23.427372+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":186,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.431731+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:42:23.431731+0000","last_peered":"2024-08-17T17:42:23.431731+0000","last_clean":"2024-08-17T17:42:23.431731+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:42:23.431731+0000","last_undegraded":"2024-08-17T17:42:23.431731+0000","last_fullsized":"2024-08-17T17:42:23.431731+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":627,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477230+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:42:32.477230+0000","last_peered":"2024-08-17T17:42:32.477230+0000","last_clean":"2024-08-17T17:42:32.477230+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:42:32.477230+0000","last_undegraded":"2024-08-17T17:42:32.477230+0000","last_fullsized":"2024-08-17T17:42:32.477230+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":685,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.487372+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:42:32.487372+0000","last_peered":"2024-08-17T17:42:32.487372+0000","last_clean":"2024-08-17T17:42:32.487372+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:42:32.487372+0000","last_undegraded":"2024-08-17T17:42:32.487372+0000","last_fullsized":"2024-08-17T17:42:32.487372+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":676,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.475265+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:42:32.475265+0000","last_peered":"2024-08-17T17:42:32.475265+0000","last_clean":"2024-08-17T17:42:32.475265+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:42:32.475265+0000","last_undegraded":"2024-08-17T17:42:32.475265+0000","last_fullsized":"2024-08-17T17:42:32.475265+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430589+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:42:23.430589+0000","last_peered":"2024-08-17T17:42:23.430589+0000","last_clean":"2024-08-17T17:42:23.430589+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:42:23.430589+0000","last_undegraded":"2024-08-17T17:42:23.430589+0000","last_fullsized":"2024-08-17T17:42:23.430589+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":680,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.475963+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:42:32.475963+0000","last_peered":"2024-08-17T17:42:32.475963+0000","last_clean":"2024-08-17T17:42:32.475963+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:42:32.475963+0000","last_undegraded":"2024-08-17T17:42:32.475963+0000","last_fullsized":"2024-08-17T17:42:32.475963+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":658,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486987+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:42:32.486987+0000","last_peered":"2024-08-17T17:42:32.486987+0000","last_clean":"2024-08-17T17:42:32.486987+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:42:32.486987+0000","last_undegraded":"2024-08-17T17:42:32.486987+0000","last_fullsized":"2024-08-17T17:42:32.486987+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":662,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.472806+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:42:32.472806+0000","last_peered":"2024-08-17T17:42:32.472806+0000","last_clean":"2024-08-17T17:42:32.472806+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:42:32.472806+0000","last_undegraded":"2024-08-17T17:42:32.472806+0000","last_fullsized":"2024-08-17T17:42:32.472806+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":677,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.475203+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:42:32.475203+0000","last_peered":"2024-08-17T17:42:32.475203+0000","last_clean":"2024-08-17T17:42:32.475203+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:42:32.475203+0000","last_undegraded":"2024-08-17T17:42:32.475203+0000","last_fullsized":"2024-08-17T17:42:32.475203+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":293,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436728+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:42:23.436728+0000","last_peered":"2024-08-17T17:42:23.436728+0000","last_clean":"2024-08-17T17:42:23.436728+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:42:23.436728+0000","last_undegraded":"2024-08-17T17:42:23.436728+0000","last_fullsized":"2024-08-17T17:42:23.436728+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":190,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.474265+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:42:32.474265+0000","last_peered":"2024-08-17T17:42:32.474265+0000","last_clean":"2024-08-17T17:42:32.474265+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:42:32.474265+0000","last_undegraded":"2024-08-17T17:42:32.474265+0000","last_fullsized":"2024-08-17T17:42:32.474265+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":53,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.486383+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:42:32.486383+0000","last_peered":"2024-08-17T17:42:32.486383+0000","last_clean":"2024-08-17T17:42:32.486383+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:42:32.486383+0000","last_undegraded":"2024-08-17T17:42:32.486383+0000","last_fullsized":"2024-08-17T17:42:32.486383+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":293,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436920+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:42:23.436920+0000","last_peered":"2024-08-17T17:42:23.436920+0000","last_clean":"2024-08-17T17:42:23.436920+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:42:23.436920+0000","last_undegraded":"2024-08-17T17:42:23.436920+0000","last_fullsized":"2024-08-17T17:42:23.436920+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":190,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430094+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:42:23.430094+0000","last_peered":"2024-08-17T17:42:23.430094+0000","last_clean":"2024-08-17T17:42:23.430094+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:42:23.430094+0000","last_undegraded":"2024-08-17T17:42:23.430094+0000","last_fullsized":"2024-08-17T17:42:23.430094+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436643+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:42:23.436643+0000","last_peered":"2024-08-17T17:42:23.436643+0000","last_clean":"2024-08-17T17:42:23.436643+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:42:23.436643+0000","last_undegraded":"2024-08-17T17:42:23.436643+0000","last_fullsized":"2024-08-17T17:42:23.436643+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":200,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.430433+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:42:23.430433+0000","last_peered":"2024-08-17T17:42:23.430433+0000","last_clean":"2024-08-17T17:42:23.430433+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:42:23.430433+0000","last_undegraded":"2024-08-17T17:42:23.430433+0000","last_fullsized":"2024-08-17T17:42:23.430433+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":682,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.477641+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:42:32.477641+0000","last_peered":"2024-08-17T17:42:32.477641+0000","last_clean":"2024-08-17T17:42:32.477641+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:42:32.477641+0000","last_undegraded":"2024-08-17T17:42:32.477641+0000","last_fullsized":"2024-08-17T17:42:32.477641+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":683,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.475654+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:42:32.475654+0000","last_peered":"2024-08-17T17:42:32.475654+0000","last_clean":"2024-08-17T17:42:32.475654+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:42:32.475654+0000","last_undegraded":"2024-08-17T17:42:32.475654+0000","last_fullsized":"2024-08-17T17:42:32.475654+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":308,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:42:23.436066+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:42:23.436066+0000","last_peered":"2024-08-17T17:42:23.436066+0000","last_clean":"2024-08-17T17:42:23.436066+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:42:23.436066+0000","last_undegraded":"2024-08-17T17:42:23.436066+0000","last_fullsized":"2024-08-17T17:42:23.436066+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":703,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:42:32.487208+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:42:32.487208+0000","last_peered":"2024-08-17T17:42:32.487208+0000","last_clean":"2024-08-17T17:42:32.487208+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:42:32.487208+0000","last_undegraded":"2024-08-17T17:42:32.487208+0000","last_fullsized":"2024-08-17T17:42:32.487208+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4097703960,"num_objects":62527,"num_object_clones":0,"num_object_copies":187581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2927,"num_objects_misplaced":3923,"num_objects_unfound":0,"num_objects_dirty":62527,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187459,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":949,"num_bytes_recovered":62193664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12529512448,"data_stored":12529500232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126604,"ondisk_log_size":126604,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201752,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2827000,"kb_used_data":2029680,"kb_used_omap":1,"kb_used_meta":797310,"kb_avail":90918152,"statfs":{"total":95995035648,"available":93100187648,"internally_reserved":0,"allocated":2078392320,"data_stored":2076410408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":816445899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922248,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161956,"kb_used_data":771612,"kb_used_omap":7,"kb_used_meta":390328,"kb_avail":92583196,"statfs":{"total":95995035648,"available":94805192704,"internally_reserved":0,"allocated":790130688,"data_stored":788168198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":399696566},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085772,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2730476,"kb_used_data":2264420,"kb_used_omap":7,"kb_used_meta":466040,"kb_avail":91014676,"statfs":{"total":95995035648,"available":93199028224,"internally_reserved":0,"allocated":2318766080,"data_stored":2316792699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":477225401},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986662,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1497328,"kb_used_data":772200,"kb_used_omap":1,"kb_used_meta":725118,"kb_avail":92247824,"statfs":{"total":95995035648,"available":94461771776,"internally_reserved":0,"allocated":790732800,"data_stored":788758994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":742521286},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435553,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2944032,"kb_used_data":2046040,"kb_used_omap":5,"kb_used_meta":897978,"kb_avail":90801120,"statfs":{"total":95995035648,"available":92980346880,"internally_reserved":0,"allocated":2095144960,"data_stored":2093183282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":919529545},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599080,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1562144,"kb_used_data":741912,"kb_used_omap":7,"kb_used_meta":820216,"kb_avail":92183008,"statfs":{"total":95995035648,"available":94395400192,"internally_reserved":0,"allocated":759717888,"data_stored":757755559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":839901357},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762606,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3500888,"kb_used_data":2725072,"kb_used_omap":7,"kb_used_meta":775800,"kb_avail":90244264,"statfs":{"total":95995035648,"available":92410126336,"internally_reserved":0,"allocated":2790473728,"data_stored":2788518201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":794419617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926132,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1839140,"kb_used_data":1018908,"kb_used_omap":7,"kb_used_meta":820216,"kb_avail":91906012,"statfs":{"total":95995035648,"available":94111756288,"internally_reserved":0,"allocated":1043361792,"data_stored":1041396261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":839901788},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,1,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1021313024,"data_stored":1021313024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2778861568,"data_stored":2778857496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":750256128,"data_stored":750256128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2086010880,"data_stored":2086010880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":779747328,"data_stored":779747328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2290290688,"data_stored":2290286616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":776339456,"data_stored":776339456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2046693376,"data_stored":2046689304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:57.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:42:57.230 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:56 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:57.230 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:56 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:57.231 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:56 smithi175 ceph-mon[101434]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:42:57.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:57.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:57.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:57.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:57.559 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:57.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:57.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:57.727 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:57.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.864 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:57.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:57.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:57.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:58.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:42:58.112 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:58.132 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:57 smithi175 ceph-mon[101434]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:42:58.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:57 smithi064 ceph-mon[104516]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:42:58.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:57 smithi064 ceph-mon[109246]: pgmap v1262: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:42:58.234 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:58.271 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:58.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:58.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:58.403 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:58.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:58.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:58.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:58.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:58.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:58.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:58.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:58.916 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:58.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:59.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:59.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:59.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3098/175773 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:59.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3098/175773 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:59.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3098/175773 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:42:59.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.486 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:59.522 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:59.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:59.655 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:59.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:42:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:59.923 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:42:59.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:59.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:00.010 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:00.011 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:00.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:00.050 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:00.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:00.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:00.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:42:59 smithi064 ceph-mon[104516]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:00.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:42:59 smithi064 ceph-mon[109246]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:00.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.331 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:00.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:42:59 smithi175 ceph-mon[101434]: pgmap v1263: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3098/175773 objects degraded (1.763%); 3683/175773 objects misplaced (2.095%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:00.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:00.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:00.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:00.601 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:00.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:00.703 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:00.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:00.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:00.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:00.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:43:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:00.976 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:01.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:01.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:01.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:01.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:01.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:01.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:01.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:01.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:01.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:01.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:01.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:01.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:01.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:01.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:43:01.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:01.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:01.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:01.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:02.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:02.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:02.239 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:02.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:02.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:02 smithi064 ceph-mon[104516]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/164826 objects degraded (1.998%); 3417/164826 objects misplaced (2.073%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:02.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:02.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:02.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:02.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:02.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:02 smithi175 ceph-mon[101434]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/164826 objects degraded (1.998%); 3417/164826 objects misplaced (2.073%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:02.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:02 smithi064 ceph-mon[109246]: pgmap v1264: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/164826 objects degraded (1.998%); 3417/164826 objects misplaced (2.073%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:02.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:02.779 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:02.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:02.912 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:03.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:03.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:03.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:03.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.186 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:03.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:03.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:03.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:03.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:03.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:03.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.864 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.966 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:03.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:04.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:04.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:03] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-08-17T17:43:04.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:04.252 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:04.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:04.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:04.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:04.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.521 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:04.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:04.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:04.659 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:04.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:04.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:04 smithi175 ceph-mon[101434]: pgmap v1265: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/163335 objects degraded (2.017%); 3417/163335 objects misplaced (2.092%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:04.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:04 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3294/163335 objects degraded (2.017%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:04 smithi064 ceph-mon[104516]: pgmap v1265: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/163335 objects degraded (2.017%); 3417/163335 objects misplaced (2.092%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:04 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3294/163335 objects degraded (2.017%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:04.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:04 smithi064 ceph-mon[109246]: pgmap v1265: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 3294/163335 objects degraded (2.017%); 3417/163335 objects misplaced (2.092%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:04.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:04 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3294/163335 objects degraded (2.017%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:04.761 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:04.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:04.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:04.914 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:05.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:05.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:05.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:05.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:05.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:05.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:05.339 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:05.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:05.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:05.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:05.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:05.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:05.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:05.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:05.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:43:05.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:05.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:06.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:06.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:06.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:06.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.196 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:06.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:06.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:06.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:06.439 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:06.456 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:06.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:06.467 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:06.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:06.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:06.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:06 smithi064 ceph-mon[104516]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3294/160467 objects degraded (2.053%); 3417/160467 objects misplaced (2.129%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:06.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:06 smithi064 ceph-mon[109246]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3294/160467 objects degraded (2.053%); 3417/160467 objects misplaced (2.129%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:06.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:06 smithi175 ceph-mon[101434]: pgmap v1266: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3294/160467 objects degraded (2.053%); 3417/160467 objects misplaced (2.129%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:43:06.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:06.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:06.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:06.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:07.042 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:07.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:07.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:07.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:07.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:07.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:07.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:07.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:07.584 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:07.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:07.696 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:43:07.696 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 3412/152103 objects degraded (2.243%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1771,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":403,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915895,"num_in_osds":8,"osd_in_since":1723915756,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":98},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":50701,"data_bytes":3325630461,"bytes_used":16260354048,"bytes_avail":751699931136,"bytes_total":767960285184,"degraded_objects":3412,"degraded_total":152103,"degraded_ratio":0.022432167675851232,"misplaced_objects":3204,"misplaced_total":152103,"misplaced_ratio":0.021064673280605906,"recovering_objects_per_sec":20,"recovering_bytes_per_sec":1346591,"recovering_keys_per_sec":0,"num_objects_recovered":234,"num_bytes_recovered":15335424,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":886},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":30,"modified":"2024-08-17T17:42:27.281728+0000","services":{}},"progress_events":{}} 2024-08-17T17:43:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:07.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:07.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:07.846 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:07.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.993 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:07.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:08.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:08.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:08.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:08.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:08.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:08.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:08.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:08.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:08 smithi064 ceph-mon[104516]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:43:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2429089519' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:08 smithi064 ceph-mon[109246]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:43:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2429089519' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:08.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:08 smithi175 ceph-mon[101434]: pgmap v1267: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:43:08.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2429089519' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:08.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:08.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:08.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:08.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:08.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:08.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:08.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:09.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:09.099 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:09.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:09.235 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:09.337 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:09.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:09.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.504 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:09.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:09.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:09.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:09.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:09.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:09 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3412/152103 objects degraded (2.243%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:09 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3412/152103 objects degraded (2.243%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:09 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3412/152103 objects degraded (2.243%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:09.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:09.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:09.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:09.883 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:43:09.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:09.935 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:10.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:10.081 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:10.183 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:10.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:10.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:10.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:10.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:10.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:10.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:10.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:10 smithi175 ceph-mon[101434]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:43:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:10.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[104516]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:43:10.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:10.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:10.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[109246]: pgmap v1268: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 3412/152103 objects degraded (2.243%); 3204/152103 objects misplaced (2.106%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:43:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:10.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:10.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:10.873 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.882 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:10.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:11.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:11.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:11.413 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:43:11.430 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:11 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2829145544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:11 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2829145544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:11 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2829145544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:11.700 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.729 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.831 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:11.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:11.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:11.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:12.002 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:12.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:12.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:12.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:12.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:12.297 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:43:12.297 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:12.335 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:43:12.336 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:12.374 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:43:12.375 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:12.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:12.419 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:43:12.419 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:12.431 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:12.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:43:12.451 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:12.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:43:12.491 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:12.528 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:43:12.528 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:12.532 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:12.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:12.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:12.564 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:43:12.565 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:12 smithi175 ceph-mon[101434]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 3638/141678 objects degraded (2.568%); 2917/141678 objects misplaced (2.059%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:43:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:12 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:12 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:12 smithi175 ceph-mon[101434]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:43:12.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3614176978' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:12.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[104516]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 3638/141678 objects degraded (2.568%); 2917/141678 objects misplaced (2.059%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:43:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[104516]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3614176978' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[109246]: pgmap v1269: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 3638/141678 objects degraded (2.568%); 2917/141678 objects misplaced (2.059%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[109246]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:43:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3614176978' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:12.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:12.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:12.823 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:12.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:12.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.971 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:12.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:13.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:13.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:13.213 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:13.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:13.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.384 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:13.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:13.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:13.521 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:13.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.661 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:13.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:13.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:13.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:13.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:13.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:13.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:14.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:14.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:14.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:14.215 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:14.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:14.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:14.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:14.403 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:13] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:43:14.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:14.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:14.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:14.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:14.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:14 smithi175 ceph-mon[101434]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 3638/140241 objects degraded (2.594%); 2917/140241 objects misplaced (2.080%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:14.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:14 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3638/140241 objects degraded (2.594%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:14 smithi064 ceph-mon[104516]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 3638/140241 objects degraded (2.594%); 2917/140241 objects misplaced (2.080%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:14.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:14 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3638/140241 objects degraded (2.594%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:14 smithi064 ceph-mon[109246]: pgmap v1271: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 3638/140241 objects degraded (2.594%); 2917/140241 objects misplaced (2.080%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:14.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:14 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3638/140241 objects degraded (2.594%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:14.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:14.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:14.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.894 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:14.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:14.983 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:14.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:15.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:15.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:15.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:15.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:15.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:43:15.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:15.444 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:15.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:15.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.610 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:15.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:15.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:15.743 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:15.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:15.868 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:15.970 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:16.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:16.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:16.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:16.247 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:16.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:16.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:16.422 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:16.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:16 smithi064 ceph-mon[104516]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 3638/137070 objects degraded (2.654%); 2917/137070 objects misplaced (2.128%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:16.481 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:16 smithi064 ceph-mon[109246]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 3638/137070 objects degraded (2.654%); 2917/137070 objects misplaced (2.128%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:16.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:43:16.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:16.557 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:16.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:16 smithi175 ceph-mon[101434]: pgmap v1272: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 3638/137070 objects degraded (2.654%); 2917/137070 objects misplaced (2.128%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:43:16.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:16.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:16.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:16.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:16.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:16.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:17.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:17.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:17.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:17.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:17.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:17.399 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:17.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:17.534 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:17.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:17.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:17.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:17.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:17.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:17.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.961 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:17.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:18.062 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:18.095 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:18.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:18.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:18.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:18.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:18.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:18.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:18.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:18.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:18 smithi064 ceph-mon[104516]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:18 smithi064 ceph-mon[109246]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:18.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:18.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:18.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:18.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:18 smithi175 ceph-mon[101434]: pgmap v1273: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:18.912 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:19.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:19.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:19.151 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:19.181 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:19.282 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:19.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:19.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:19.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:19.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:19.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:19 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 3827/127266 objects degraded (3.007%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:19.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:19 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 3827/127266 objects degraded (3.007%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:19.603 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:19.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:19.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:19.843 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:19.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:19.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:19.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:19 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 3827/127266 objects degraded (3.007%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:19.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:20.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:20.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:20.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:20.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:20.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:20.416 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221001 8.21 8:8407315b:::benchmark_data_smithi064_242541_object98508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.78554967299999, 2024-08-17T17:43:20.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.682936939, 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221001 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:43:20.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4499999999999997e-07 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732147+0000", 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.269e-06 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732156+0000", 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0149999999999992e-06 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732160+0000", 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3449999999999999e-06 2024-08-17T17:43:20.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732206+0000", 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6066000000000002e-05 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732213+0000", 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9280000000000004e-06 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351505+0000", 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61929169299999998 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351518+0000", 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2826000000000001e-05 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409384+0000", 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057866347 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409504+0000", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119895 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409764+0000", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026007200000000003 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.411743+0000", 2024-08-17T17:43:20.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019789 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414636+0000", 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002893161 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414860+0000", 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000223816 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414939+0000", 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8861000000000003e-05 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414991+0000", 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2759000000000001e-05 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415080+0000", 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8341e-05 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124576 9.5 9:a86362b4:::benchmark_data_smithi064_275711_object124575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.959422304, 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2467486480000001, 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.516 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124576 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558271+0000", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.269e-06 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558291+0000", 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9082000000000001e-05 2024-08-17T17:43:20.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558306+0000", 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5187999999999999e-05 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558311+0000", 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2889999999999997e-06 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558781+0000", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047030999999999999 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558984+0000", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020240899999999999 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.559149+0000", 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016526299999999999 2024-08-17T17:43:20.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.563915+0000", 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047659030000000002 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.569779+0000", 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058636549999999997 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601496+0000", 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031717707999999997 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601503+0000", 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0729999999999999e-06 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601309+0000", 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99980544400000004 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601326+0000", 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7337999999999999e-05 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.804957+0000", 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2036309160000001 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.805019+0000", 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1801e-05 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124594 9.5 9:a480dbe2:::benchmark_data_smithi064_275711_object124593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.941928752, 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2331379170000001, 2024-08-17T17:43:20.524 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124594 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575765+0000", 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0690000000000001e-06 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575786+0000", 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1311999999999999e-05 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575798+0000", 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1841000000000001e-05 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575806+0000", 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9170000000000006e-06 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575873+0000", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6804999999999995e-05 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.576025+0000", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015275299999999999 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.576172+0000", 2024-08-17T17:43:20.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014673099999999999 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.581021+0000", 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048487820000000003 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.586844+0000", 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058226750000000002 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601612+0000", 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014768325000000001 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601623+0000", 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0984e-05 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601380+0000", 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.999756914 2024-08-17T17:43:20.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601391+0000", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1615e-05 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808792+0000", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.207400504 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808902+0000", 2024-08-17T17:43:20.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010969 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124652 9.5 9:aa47bb3e:::benchmark_data_smithi064_275711_object124651:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.867455703, 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1929173679999998, 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.532 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124652 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650238+0000", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7e-06 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650263+0000", 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4824000000000002e-05 2024-08-17T17:43:20.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650279+0000", 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6161999999999999e-05 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650288+0000", 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1540000000000002e-06 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650364+0000", 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6188999999999998e-05 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650376+0000", 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2605999999999999e-05 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601433+0000", 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95105666799999999 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601441+0000", 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8639999999999996e-06 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808956+0000", 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2075150640000001 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.809085+0000", 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012875399999999999 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.809245+0000", 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016023799999999999 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.819892+0000", 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010647301 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822932+0000", 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030402010000000002 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837245+0000", 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014312147000000001 2024-08-17T17:43:20.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837310+0000", 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5772999999999995e-05 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843154+0000", 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058437230000000003 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124655 9.5 9:a81f9d47:::benchmark_data_smithi064_275711_object124654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.865286572, 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1968849239999999, 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124655 2024-08-17T17:43:20.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652407+0000", 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018e-06 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652436+0000", 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.934e-05 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652447+0000", 2024-08-17T17:43:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1221e-05 2024-08-17T17:43:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652455+0000", 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652498+0000", 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3609000000000002e-05 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652505+0000", 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6309999999999997e-06 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601475+0000", 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94897038600000005 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601485+0000", 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2059999999999996e-06 2024-08-17T17:43:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.812912+0000", 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2114278860000001 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.813067+0000", 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015408300000000001 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.813257+0000", 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019044299999999999 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822946+0000", 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096887520000000001 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822986+0000", 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0778000000000002e-05 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843268+0000", 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020281995000000001 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843308+0000", 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9626000000000003e-05 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849291+0000", 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059826929999999999 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124687 9.5 9:a52614fa:::benchmark_data_smithi064_275711_object124686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.832564449, 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1708211930000001, 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.551 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124687 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:43:20.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685129+0000", 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4600000000000004e-07 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685169+0000", 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0126000000000002e-05 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685178+0000", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1200000000000008e-06 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685184+0000", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000003e-06 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685356+0000", 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171733 2024-08-17T17:43:20.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685366+0000", 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0509999999999999e-05 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601513+0000", 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91614678299999996 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601518+0000", 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9640000000000002e-06 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816519+0000", 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.215000675 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816637+0000", 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118484 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816738+0000", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101216 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826294+0000", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095560009999999997 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829584+0000", 2024-08-17T17:43:20.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032897830000000001 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849468+0000", 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019883596 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849530+0000", 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1863e-05 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.855949+0000", 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064195329999999998 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124690 9.5 9:a653a86d:::benchmark_data_smithi064_275711_object124689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.82622237, 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1710445190000001, 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.559 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124690 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691472+0000", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691495+0000", 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.298e-05 2024-08-17T17:43:20.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691516+0000", 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1622999999999999e-05 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691525+0000", 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2130000000000008e-06 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691583+0000", 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8037e-05 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691592+0000", 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8570000000000007e-06 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601548+0000", 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90995501899999998 2024-08-17T17:43:20.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601555+0000", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6199999999999999e-06 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.819932+0000", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218377241 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.820049+0000", 2024-08-17T17:43:20.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116387 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.820150+0000", 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101167 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829683+0000", 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095330499999999995 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837074+0000", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073908680000000001 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856054+0000", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01898004 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856113+0000", 2024-08-17T17:43:20.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9218999999999997e-05 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862515+0000", 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064015030000000002 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124707 9.5 9:a946d6e3:::benchmark_data_smithi064_275711_object124706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.80956062200001, 2024-08-17T17:43:20.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.164556192, 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124707 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708133+0000", 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.544e-06 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708156+0000", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665999999999999e-05 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708173+0000", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6824999999999999e-05 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708181+0000", 2024-08-17T17:43:20.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9729999999999997e-06 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708243+0000", 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2565999999999997e-05 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708254+0000", 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0761000000000001e-05 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601579+0000", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89332489000000004 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601583+0000", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4050000000000002e-06 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823020+0000", 2024-08-17T17:43:20.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2214364769999999 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823128+0000", 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108113 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823232+0000", 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010371500000000001 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837172+0000", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013940111999999999 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843169+0000", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059975000000000002 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.866982+0000", 2024-08-17T17:43:20.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023812887000000001 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.867046+0000", 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4158000000000005e-05 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.872688+0000", 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056416000000000001 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124759 9.5 9:a81bd317:::benchmark_data_smithi064_275711_object124758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.755592627, 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.126767981, 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.576 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124759 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762101+0000", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.483e-06 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762130+0000", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8365e-05 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762146+0000", 2024-08-17T17:43:20.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6033999999999999e-05 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762154+0000", 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2470000000000008e-06 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762216+0000", 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2317000000000007e-05 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762227+0000", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063000000000001e-05 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601611+0000", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83938328900000003 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601618+0000", 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4850000000000003e-06 2024-08-17T17:43:20.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826349+0000", 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2247311299999999 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826489+0000", 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139801 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826594+0000", 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105067 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843174+0000", 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016579611000000001 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843226+0000", 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2272999999999999e-05 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.880086+0000", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036859801999999997 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.880158+0000", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2336999999999995e-05 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888868+0000", 2024-08-17T17:43:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087096770000000007 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124783 9.5 9:a6c522ba:::benchmark_data_smithi064_275711_object124782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.728766038, 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1068486069999999, 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124783 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788928+0000", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.48e-06 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788950+0000", 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2152000000000001e-05 2024-08-17T17:43:20.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788968+0000", 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7768999999999999e-05 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788976+0000", 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0010000000000001e-06 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.789045+0000", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9276000000000006e-05 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.789055+0000", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8300000000000008e-06 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601642+0000", 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81258676100000005 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601646+0000", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3440000000000003e-06 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829716+0000", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.228070405 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829851+0000", 2024-08-17T17:43:20.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000134363 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.830008+0000", 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157267 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843178+0000", 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013170125 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849395+0000", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062171759999999996 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888981+0000", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039585596000000001 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.889041+0000", 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0136000000000002e-05 2024-08-17T17:43:20.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895775+0000", 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067339260000000003 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124826 9.5 9:ae6b922f:::benchmark_data_smithi064_275711_object124825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.681347544, 2024-08-17T17:43:20.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0662265, 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124826 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:43:20.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836347+0000", 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9379999999999999e-06 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836375+0000", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8479999999999998e-05 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836397+0000", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1815e-05 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836408+0000", 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0366999999999999e-05 2024-08-17T17:43:20.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836473+0000", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5469000000000006e-05 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836485+0000", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181e-05 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601666+0000", 2024-08-17T17:43:20.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76518151599999995 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601671+0000", 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3830000000000002e-06 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.833771+0000", 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.232100143 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.833943+0000", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000172549 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.834055+0000", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111354 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849307+0000", 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015252474 2024-08-17T17:43:20.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856008+0000", 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067006920000000003 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895851+0000", 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039842620000000002 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895919+0000", 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8134999999999997e-05 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902571+0000", 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066527549999999998 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.600 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124834 9.5 9:a84b73a1:::benchmark_data_smithi064_275711_object124833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.671769303, 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.063460681, 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124834 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:43:20.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845925+0000", 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.719e-06 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845976+0000", 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0979000000000001e-05 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845991+0000", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4711e-05 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846000+0000", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1179999999999998e-06 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846084+0000", 2024-08-17T17:43:20.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4714000000000002e-05 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846097+0000", 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2904999999999999e-05 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601692+0000", 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75559463699999996 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601696+0000", 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0029999999999996e-06 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862556+0000", 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.260859795 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862739+0000", 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018281800000000001 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862930+0000", 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000191917 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888940+0000", 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026009211000000001 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902593+0000", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013653439 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902638+0000", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5309999999999998e-05 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902687+0000", 2024-08-17T17:43:20.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8667e-05 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909384+0000", 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066967379999999998 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124848 9.5 9:a16a5733:::benchmark_data_smithi064_275711_object124847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.65696076, 2024-08-17T17:43:20.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1062764290000002, 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124848 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:43:20.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860733+0000", 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.536e-06 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860755+0000", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1763000000000002e-05 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860773+0000", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7655e-05 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860780+0000", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5429999999999996e-06 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860848+0000", 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7397999999999994e-05 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860862+0000", 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.47e-05 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601726+0000", 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74086384599999999 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601735+0000", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4339999999999992e-06 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909452+0000", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3077175620000001 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909623+0000", 2024-08-17T17:43:20.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017124100000000001 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909785+0000", 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016205200000000001 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922612+0000", 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012826561 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922629+0000", 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7314000000000001e-05 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.960337+0000", 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037707209999999998 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.960397+0000", 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0488000000000001e-05 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967008+0000", 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066111260000000002 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124851 9.5 9:a13810df:::benchmark_data_smithi064_275711_object124850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.653931089, 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.110583149, 2024-08-17T17:43:20.617 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124851 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:43:20.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863762+0000", 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.09e-07 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863802+0000", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0132999999999997e-05 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863814+0000", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1408e-05 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863819+0000", 2024-08-17T17:43:20.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9189999999999998e-06 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863872+0000", 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3310999999999998e-05 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863878+0000", 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3779999999999998e-06 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601806+0000", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73792795099999997 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601811+0000", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9230000000000001e-06 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914048+0000", 2024-08-17T17:43:20.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3122370210000001 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914205+0000", 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000156334 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914367+0000", 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162497 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922635+0000", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082677570000000006 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929492+0000", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006857096 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967067+0000", 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037574514000000003 2024-08-17T17:43:20.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967129+0000", 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2793999999999995e-05 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974345+0000", 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072152040000000002 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124854 9.5 9:a175789f:::benchmark_data_smithi064_275711_object124853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.649585101, 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1131798100000001, 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.626 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124854 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868109+0000", 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156e-06 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868147+0000", 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8748000000000002e-05 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868161+0000", 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3872e-05 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868169+0000", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0129999999999993e-06 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868229+0000", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0275000000000001e-05 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868241+0000", 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1501999999999999e-05 2024-08-17T17:43:20.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601831+0000", 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73359040200000003 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601836+0000", 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9409999999999998e-06 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918294+0000", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.316458159 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918435+0000", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140469 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918604+0000", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168839 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929552+0000", 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010947811999999999 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929564+0000", 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2254e-05 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974488+0000", 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044924015999999997 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974556+0000", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8674e-05 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981287+0000", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067306780000000004 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124857 9.5 9:aba14c59:::benchmark_data_smithi064_275711_object124856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:43:20.633 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.644501433, 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1145436690000001, 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124857 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873193+0000", 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.159e-06 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873235+0000", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1999000000000002e-05 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873250+0000", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4788e-05 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873257+0000", 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8220000000000001e-06 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873311+0000", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4478000000000003e-05 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873319+0000", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1540000000000002e-06 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601872+0000", 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72855236599999995 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601876+0000", 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0940000000000001e-06 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922661+0000", 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3207853940000001 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922808+0000", 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014638499999999999 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922968+0000", 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159889 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929585+0000", 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066178030000000002 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.931456+0000", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001870771 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981424+0000", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049967984 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981488+0000", 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3699000000000001e-05 2024-08-17T17:43:20.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987735+0000", 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062468840000000003 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124863 9.5 9:a02aa46e:::benchmark_data_smithi064_275711_object124862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.621888923, 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0986319409999998, 2024-08-17T17:43:20.641 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124863 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895805+0000", 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.773e-06 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895826+0000", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0367999999999999e-05 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895840+0000", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4810999999999999e-05 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895847+0000", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7560000000000002e-06 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895913+0000", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5739000000000002e-05 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895921+0000", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601901+0000", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70597918100000001 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:20.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601906+0000", 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5099999999999998e-06 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926428+0000", 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3245222110000001 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926595+0000", 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016628699999999999 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926762+0000", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167252 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.933120+0000", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063576300000000004 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.944327+0000", 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011207205 2024-08-17T17:43:20.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987802+0000", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.043475371999999998 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987925+0000", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122485 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.994435+0000", 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065108700000000002 2024-08-17T17:43:20.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:141488 9.5 9:a257511e:::benchmark_data_smithi064_275711_object16493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:53.424456+0000", 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 86.093236105000003, 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27408382199999998, 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.649 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 141488 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424456+0000", 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424456+0000", 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424458+0000", 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7540000000000001e-06 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424463+0000", 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6290000000000001e-06 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424479+0000", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6169999999999999e-05 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.424487+0000", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3110000000000006e-06 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.433367+0000", 2024-08-17T17:43:20.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088796970000000006 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.433490+0000", 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012288300000000001 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.637253+0000", 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.203763691 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.637530+0000", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027629999999999999 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.637923+0000", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039283100000000001 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.649191+0000", 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011268271 2024-08-17T17:43:20.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.692074+0000", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042882937000000003 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.692125+0000", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0787e-05 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.692224+0000", 2024-08-17T17:43:20.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9809000000000006e-05 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:53.698540+0000", 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006315752 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:164147 9.5 9:a37b5145:::benchmark_data_smithi064_275711_object39152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:21.912724+0000", 2024-08-17T17:43:20.656 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.604968632999999, 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27520264300000002, 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164147 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912724+0000", 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912724+0000", 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912726+0000", 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9769999999999999e-06 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912733+0000", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9249999999999998e-06 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912752+0000", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9582e-05 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.912772+0000", 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9661e-05 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.981211+0000", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068438531999999996 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:21.981516+0000", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030592400000000001 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.127565+0000", 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.146048395 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.127672+0000", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107143 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.127833+0000", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016150600000000001 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.132615+0000", 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047812009999999997 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.164465+0000", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031850825999999999 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.181567+0000", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017101522000000001 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.181632+0000", 2024-08-17T17:43:20.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5384999999999998e-05 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:22.187926+0000", 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062940640000000003 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:167194 9.5 9:af8793be:::benchmark_data_smithi064_275711_object42199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:42:25.874411+0000", 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 53.643281074000001, 2024-08-17T17:43:20.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30834493499999999, 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 167194 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874411+0000", 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874411+0000", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874413+0000", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9250000000000002e-06 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874419+0000", 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1959999999999999e-06 2024-08-17T17:43:20.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874435+0000", 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6379e-05 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.874442+0000", 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4760000000000001e-06 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.884655+0000", 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010212834 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.884823+0000", 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016766400000000001 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.990759+0000", 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10593645 2024-08-17T17:43:20.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.990829+0000", 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9608000000000002e-05 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.990947+0000", 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117608 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:20.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:25.996692+0000", 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057454949999999998 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:26.028144+0000", 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031451814000000002 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:26.177331+0000", 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14918741799999999 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:26.177406+0000", 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4966e-05 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:42:26.182756+0000", 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053501019999999998 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:20.670 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:20.671 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:43:20.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:20.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:20 smithi064 ceph-mon[104516]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:20.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:20 smithi064 ceph-mon[109246]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:20.701 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:20.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:20.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:20.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:20 smithi175 ceph-mon[101434]: pgmap v1274: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 3827/127266 objects degraded (3.007%); 2655/127266 objects misplaced (2.086%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:43:20.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:20.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:21.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:21.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:21.229 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:21.267 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:21.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:21.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:21.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:21.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:21.660 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:21.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:21.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:21.841 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:21.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:21.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:22.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:22.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:22.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:22.266 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:22.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:22.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:22.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:22.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:22.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:22.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.681 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:22.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:22 smithi064 ceph-mon[104516]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4037/116250 objects degraded (3.473%); 2372/116250 objects misplaced (2.040%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:43:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:22 smithi064 ceph-mon[109246]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4037/116250 objects degraded (3.473%); 2372/116250 objects misplaced (2.040%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:43:22.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:22.808 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:22.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:22.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:22.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:22 smithi175 ceph-mon[101434]: pgmap v1275: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4037/116250 objects degraded (3.473%); 2372/116250 objects misplaced (2.040%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:43:22.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:23.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:23.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:23.185 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.227 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:23.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:23.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:23.367 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:23.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:23.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:23.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:23.630 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:23.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:23.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4037/116250 objects degraded (3.473%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:23.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4037/116250 objects degraded (3.473%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:23.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:23.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:23.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:23.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4037/116250 objects degraded (3.473%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:23.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:24.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:24.083 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:24.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:43:24.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:24.200 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:23] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-08-17T17:43:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.239 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:24.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:24.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:24.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:24.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:24.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[104516]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4037/114705 objects degraded (3.519%); 2372/114705 objects misplaced (2.068%); 785 KiB/s, 12 objects/s recovering 2024-08-17T17:43:24.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:24.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:24.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:24.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:24.600 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:24.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:24.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:24.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[109246]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4037/114705 objects degraded (3.519%); 2372/114705 objects misplaced (2.068%); 785 KiB/s, 12 objects/s recovering 2024-08-17T17:43:24.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:24.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:24 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:24.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:24.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:24.901 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:24 smithi175 ceph-mon[101434]: pgmap v1276: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4037/114705 objects degraded (3.519%); 2372/114705 objects misplaced (2.068%); 785 KiB/s, 12 objects/s recovering 2024-08-17T17:43:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:24.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:24 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:25.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:25.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:25.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:25.180 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:25.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:25.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:25.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:25.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:25.471 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:25.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:25.601 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:25.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:25.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:25.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:25.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:25.775 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:43:25.776 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:43:25.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:25.877 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:25.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:25.931 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:25.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:26.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:26.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:26.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:26.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:26.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:26.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:26.422 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:26.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:26.556 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:26.639 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:26.658 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:26.664 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1277,"stamp":"2024-08-17T17:43:25.314905+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2440370173,"num_objects":37193,"num_object_clones":0,"num_object_copies":111579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4037,"num_objects_misplaced":2372,"num_objects_unfound":0,"num_objects_dirty":37193,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":213729,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1511,"num_bytes_recovered":99490019,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129138,"ondisk_log_size":129138,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13388276,"kb_used_data":7621620,"kb_used_omap":46,"kb_used_meta":5766481,"kb_avail":736572940,"statfs":{"total":767960285184,"available":754250690560,"internally_reserved":0,"allocated":7804538880,"data_stored":7788739634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5904876787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,4,11,14,3,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-626130944,"num_objects":-9554,"num_object_clones":0,"num_object_copies":-28662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":399,"num_objects_misplaced":-545,"num_objects_unfound":0,"num_objects_dirty":-9554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9554,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006239"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":669,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422672+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:43:11.422672+0000","last_peered":"2024-08-17T17:43:11.422672+0000","last_clean":"2024-08-17T17:43:11.422672+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:43:11.422672+0000","last_undegraded":"2024-08-17T17:43:11.422672+0000","last_fullsized":"2024-08-17T17:43:11.422672+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":405,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.929618+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:42:55.929618+0000","last_peered":"2024-08-17T17:42:55.929618+0000","last_clean":"2024-08-17T17:42:55.929618+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:42:55.929618+0000","last_undegraded":"2024-08-17T17:42:55.929618+0000","last_fullsized":"2024-08-17T17:42:55.929618+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417856+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:43:11.417856+0000","last_peered":"2024-08-17T17:43:11.417856+0000","last_clean":"2024-08-17T17:43:11.417856+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:43:11.417856+0000","last_undegraded":"2024-08-17T17:43:11.417856+0000","last_fullsized":"2024-08-17T17:43:11.417856+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":661,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417110+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:43:11.417110+0000","last_peered":"2024-08-17T17:43:11.417110+0000","last_clean":"2024-08-17T17:43:11.417110+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:43:11.417110+0000","last_undegraded":"2024-08-17T17:43:11.417110+0000","last_fullsized":"2024-08-17T17:43:11.417110+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":660,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419790+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:43:11.419790+0000","last_peered":"2024-08-17T17:43:11.419790+0000","last_clean":"2024-08-17T17:43:11.419790+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:43:11.419790+0000","last_undegraded":"2024-08-17T17:43:11.419790+0000","last_fullsized":"2024-08-17T17:43:11.419790+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":391,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.930063+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:42:55.930063+0000","last_peered":"2024-08-17T17:42:55.930063+0000","last_clean":"2024-08-17T17:42:55.930063+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:42:55.930063+0000","last_undegraded":"2024-08-17T17:42:55.930063+0000","last_fullsized":"2024-08-17T17:42:55.930063+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":669,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.427372+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:43:11.427372+0000","last_peered":"2024-08-17T17:43:11.427372+0000","last_clean":"2024-08-17T17:43:11.427372+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:43:11.427372+0000","last_undegraded":"2024-08-17T17:43:11.427372+0000","last_fullsized":"2024-08-17T17:43:11.427372+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":339,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.437921+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:43:11.437921+0000","last_peered":"2024-08-17T17:43:11.437921+0000","last_clean":"2024-08-17T17:43:11.437921+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:43:11.437921+0000","last_undegraded":"2024-08-17T17:43:11.437921+0000","last_fullsized":"2024-08-17T17:43:11.437921+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"404'13248","reported_seq":16100,"reported_epoch":404,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:21.086640+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:43:21.086640+0000","last_peered":"2024-08-17T17:43:21.086640+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:43:21.086640+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"385'4100","ondisk_log_start":"385'4100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157024256,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4037,"num_objects_misplaced":2372,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13248,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1390,"num_bytes_recovered":91095040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":2372},{"shards":"0,5","objects":-4013}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":400,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928834+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:42:55.928834+0000","last_peered":"2024-08-17T17:42:55.928834+0000","last_clean":"2024-08-17T17:42:55.928834+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:42:55.928834+0000","last_undegraded":"2024-08-17T17:42:55.928834+0000","last_fullsized":"2024-08-17T17:42:55.928834+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":199,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917585+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:42:55.917585+0000","last_peered":"2024-08-17T17:42:55.917585+0000","last_clean":"2024-08-17T17:42:55.917585+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:42:55.917585+0000","last_undegraded":"2024-08-17T17:42:55.917585+0000","last_fullsized":"2024-08-17T17:42:55.917585+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"404'13253","reported_seq":13322,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.404788+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:43:21.404788+0000","last_peered":"2024-08-17T17:43:21.404788+0000","last_clean":"2024-08-17T17:43:21.404788+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:43:21.404788+0000","last_undegraded":"2024-08-17T17:43:21.404788+0000","last_fullsized":"2024-08-17T17:43:21.404788+0000","mapping_epoch":375,"log_start":"389'6100","ondisk_log_start":"389'6100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7153,"log_dups_size":0,"ondisk_log_size":7153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156434432,"num_objects":2387,"num_object_clones":0,"num_object_copies":7161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13253,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":401,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928771+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:42:55.928771+0000","last_peered":"2024-08-17T17:42:55.928771+0000","last_clean":"2024-08-17T17:42:55.928771+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:42:55.928771+0000","last_undegraded":"2024-08-17T17:42:55.928771+0000","last_fullsized":"2024-08-17T17:42:55.928771+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":296,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917629+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:42:55.917629+0000","last_peered":"2024-08-17T17:42:55.917629+0000","last_clean":"2024-08-17T17:42:55.917629+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:42:55.917629+0000","last_undegraded":"2024-08-17T17:42:55.917629+0000","last_fullsized":"2024-08-17T17:42:55.917629+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"404'13267","reported_seq":13306,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.193947+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:43:21.193947+0000","last_peered":"2024-08-17T17:43:21.193947+0000","last_clean":"2024-08-17T17:43:21.193947+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:43:21.193947+0000","last_undegraded":"2024-08-17T17:43:21.193947+0000","last_fullsized":"2024-08-17T17:43:21.193947+0000","mapping_epoch":375,"log_start":"385'4400","ondisk_log_start":"385'4400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8867,"log_dups_size":0,"ondisk_log_size":8867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155254784,"num_objects":2369,"num_object_clones":0,"num_object_copies":7107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13267,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":418,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928902+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:42:55.928902+0000","last_peered":"2024-08-17T17:42:55.928902+0000","last_clean":"2024-08-17T17:42:55.928902+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:42:55.928902+0000","last_undegraded":"2024-08-17T17:42:55.928902+0000","last_fullsized":"2024-08-17T17:42:55.928902+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":319,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.935872+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:42:55.935872+0000","last_peered":"2024-08-17T17:42:55.935872+0000","last_clean":"2024-08-17T17:42:55.935872+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:42:55.935872+0000","last_undegraded":"2024-08-17T17:42:55.935872+0000","last_fullsized":"2024-08-17T17:42:55.935872+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"404'13276","reported_seq":13315,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.224516+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:43:21.224516+0000","last_peered":"2024-08-17T17:43:21.224516+0000","last_clean":"2024-08-17T17:43:21.224516+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:43:21.224516+0000","last_undegraded":"2024-08-17T17:43:21.224516+0000","last_fullsized":"2024-08-17T17:43:21.224516+0000","mapping_epoch":375,"log_start":"385'4400","ondisk_log_start":"385'4400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156893184,"num_objects":2394,"num_object_clones":0,"num_object_copies":7182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13276,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":199,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918096+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:42:55.918096+0000","last_peered":"2024-08-17T17:42:55.918096+0000","last_clean":"2024-08-17T17:42:55.918096+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:42:55.918096+0000","last_undegraded":"2024-08-17T17:42:55.918096+0000","last_fullsized":"2024-08-17T17:42:55.918096+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":675,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417113+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:43:11.417113+0000","last_peered":"2024-08-17T17:43:11.417113+0000","last_clean":"2024-08-17T17:43:11.417113+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:43:11.417113+0000","last_undegraded":"2024-08-17T17:43:11.417113+0000","last_fullsized":"2024-08-17T17:43:11.417113+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"404'13557","reported_seq":13627,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:24.338677+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:43:24.338677+0000","last_peered":"2024-08-17T17:43:24.338677+0000","last_clean":"2024-08-17T17:43:24.338677+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:43:24.338677+0000","last_undegraded":"2024-08-17T17:43:24.338677+0000","last_fullsized":"2024-08-17T17:43:24.338677+0000","mapping_epoch":375,"log_start":"390'6500","ondisk_log_start":"390'6500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144769024,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13557,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":759,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.929955+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:42:55.929955+0000","last_peered":"2024-08-17T17:42:55.929955+0000","last_clean":"2024-08-17T17:42:55.929955+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:42:55.929955+0000","last_undegraded":"2024-08-17T17:42:55.929955+0000","last_fullsized":"2024-08-17T17:42:55.929955+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":690,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422174+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:43:11.422174+0000","last_peered":"2024-08-17T17:43:11.422174+0000","last_clean":"2024-08-17T17:43:11.422174+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:43:11.422174+0000","last_undegraded":"2024-08-17T17:43:11.422174+0000","last_fullsized":"2024-08-17T17:43:11.422174+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":320,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918606+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:42:55.918606+0000","last_peered":"2024-08-17T17:42:55.918606+0000","last_clean":"2024-08-17T17:42:55.918606+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:42:55.918606+0000","last_undegraded":"2024-08-17T17:42:55.918606+0000","last_fullsized":"2024-08-17T17:42:55.918606+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"404'12903","reported_seq":12973,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.378830+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:43:21.378830+0000","last_peered":"2024-08-17T17:43:21.378830+0000","last_clean":"2024-08-17T17:43:21.378830+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:43:21.378830+0000","last_undegraded":"2024-08-17T17:43:21.378830+0000","last_fullsized":"2024-08-17T17:43:21.378830+0000","mapping_epoch":375,"log_start":"388'5700","ondisk_log_start":"388'5700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7203,"log_dups_size":0,"ondisk_log_size":7203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153026560,"num_objects":2335,"num_object_clones":0,"num_object_copies":7005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12903,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":420,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917351+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:42:55.917351+0000","last_peered":"2024-08-17T17:42:55.917351+0000","last_clean":"2024-08-17T17:42:55.917351+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:42:55.917351+0000","last_undegraded":"2024-08-17T17:42:55.917351+0000","last_fullsized":"2024-08-17T17:42:55.917351+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":326,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917817+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:42:55.917817+0000","last_peered":"2024-08-17T17:42:55.917817+0000","last_clean":"2024-08-17T17:42:55.917817+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:42:55.917817+0000","last_undegraded":"2024-08-17T17:42:55.917817+0000","last_fullsized":"2024-08-17T17:42:55.917817+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":296,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918852+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:42:55.918852+0000","last_peered":"2024-08-17T17:42:55.918852+0000","last_clean":"2024-08-17T17:42:55.918852+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:42:55.918852+0000","last_undegraded":"2024-08-17T17:42:55.918852+0000","last_fullsized":"2024-08-17T17:42:55.918852+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"404'13133","reported_seq":13172,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:20.818808+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:43:20.818808+0000","last_peered":"2024-08-17T17:43:20.818808+0000","last_clean":"2024-08-17T17:43:20.818808+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:43:20.818808+0000","last_undegraded":"2024-08-17T17:43:20.818808+0000","last_fullsized":"2024-08-17T17:43:20.818808+0000","mapping_epoch":375,"log_start":"390'6138","ondisk_log_start":"390'6138","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13133,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":296,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917563+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:42:55.917563+0000","last_peered":"2024-08-17T17:42:55.917563+0000","last_clean":"2024-08-17T17:42:55.917563+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:42:55.917563+0000","last_undegraded":"2024-08-17T17:42:55.917563+0000","last_fullsized":"2024-08-17T17:42:55.917563+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":311,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.919175+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:42:55.919175+0000","last_peered":"2024-08-17T17:42:55.919175+0000","last_clean":"2024-08-17T17:42:55.919175+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:42:55.919175+0000","last_undegraded":"2024-08-17T17:42:55.919175+0000","last_fullsized":"2024-08-17T17:42:55.919175+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"404'13231","reported_seq":13301,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:24.355147+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:43:24.355147+0000","last_peered":"2024-08-17T17:43:24.355147+0000","last_clean":"2024-08-17T17:43:24.355147+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:43:24.355147+0000","last_undegraded":"2024-08-17T17:43:24.355147+0000","last_fullsized":"2024-08-17T17:43:24.355147+0000","mapping_epoch":375,"log_start":"390'6200","ondisk_log_start":"390'6200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140574720,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13231,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":420,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917391+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:42:55.917391+0000","last_peered":"2024-08-17T17:42:55.917391+0000","last_clean":"2024-08-17T17:42:55.917391+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:42:55.917391+0000","last_undegraded":"2024-08-17T17:42:55.917391+0000","last_fullsized":"2024-08-17T17:42:55.917391+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":671,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.423646+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:43:11.423646+0000","last_peered":"2024-08-17T17:43:11.423646+0000","last_clean":"2024-08-17T17:43:11.423646+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:43:11.423646+0000","last_undegraded":"2024-08-17T17:43:11.423646+0000","last_fullsized":"2024-08-17T17:43:11.423646+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":659,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.426227+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:43:11.426227+0000","last_peered":"2024-08-17T17:43:11.426227+0000","last_clean":"2024-08-17T17:43:11.426227+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:43:11.426227+0000","last_undegraded":"2024-08-17T17:43:11.426227+0000","last_fullsized":"2024-08-17T17:43:11.426227+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"404'13628","reported_seq":13698,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:24.359842+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:43:24.359842+0000","last_peered":"2024-08-17T17:43:24.359842+0000","last_clean":"2024-08-17T17:43:24.359842+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:43:24.359842+0000","last_undegraded":"2024-08-17T17:43:24.359842+0000","last_fullsized":"2024-08-17T17:43:24.359842+0000","mapping_epoch":375,"log_start":"390'6600","ondisk_log_start":"390'6600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13628,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":743,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928462+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:42:55.928462+0000","last_peered":"2024-08-17T17:42:55.928462+0000","last_clean":"2024-08-17T17:42:55.928462+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:42:55.928462+0000","last_undegraded":"2024-08-17T17:42:55.928462+0000","last_fullsized":"2024-08-17T17:42:55.928462+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":997,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917674+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:42:55.917674+0000","last_peered":"2024-08-17T17:42:55.917674+0000","last_clean":"2024-08-17T17:42:55.917674+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:42:55.917674+0000","last_undegraded":"2024-08-17T17:42:55.917674+0000","last_fullsized":"2024-08-17T17:42:55.917674+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":654,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422574+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:43:11.422574+0000","last_peered":"2024-08-17T17:43:11.422574+0000","last_clean":"2024-08-17T17:43:11.422574+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:43:11.422574+0000","last_undegraded":"2024-08-17T17:43:11.422574+0000","last_fullsized":"2024-08-17T17:43:11.422574+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918796+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:42:55.918796+0000","last_peered":"2024-08-17T17:42:55.918796+0000","last_clean":"2024-08-17T17:42:55.918796+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:42:55.918796+0000","last_undegraded":"2024-08-17T17:42:55.918796+0000","last_fullsized":"2024-08-17T17:42:55.918796+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"404'13250","reported_seq":13320,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.084720+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:43:21.084720+0000","last_peered":"2024-08-17T17:43:21.084720+0000","last_clean":"2024-08-17T17:43:21.084720+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:43:21.084720+0000","last_undegraded":"2024-08-17T17:43:21.084720+0000","last_fullsized":"2024-08-17T17:43:21.084720+0000","mapping_epoch":375,"log_start":"384'4100","ondisk_log_start":"384'4100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157810688,"num_objects":2408,"num_object_clones":0,"num_object_copies":7224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13250,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":471,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.420680+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:43:11.420680+0000","last_peered":"2024-08-17T17:43:11.420680+0000","last_clean":"2024-08-17T17:43:11.420680+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:43:11.420680+0000","last_undegraded":"2024-08-17T17:43:11.420680+0000","last_fullsized":"2024-08-17T17:43:11.420680+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":784,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928381+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:42:55.928381+0000","last_peered":"2024-08-17T17:42:55.928381+0000","last_clean":"2024-08-17T17:42:55.928381+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:42:55.928381+0000","last_undegraded":"2024-08-17T17:42:55.928381+0000","last_fullsized":"2024-08-17T17:42:55.928381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":676,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419996+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:43:11.419996+0000","last_peered":"2024-08-17T17:43:11.419996+0000","last_clean":"2024-08-17T17:43:11.419996+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:43:11.419996+0000","last_undegraded":"2024-08-17T17:43:11.419996+0000","last_fullsized":"2024-08-17T17:43:11.419996+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":673,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.420002+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:43:11.420002+0000","last_peered":"2024-08-17T17:43:11.420002+0000","last_clean":"2024-08-17T17:43:11.420002+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:43:11.420002+0000","last_undegraded":"2024-08-17T17:43:11.420002+0000","last_fullsized":"2024-08-17T17:43:11.420002+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"404'13186","reported_seq":13225,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.159273+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:43:21.159273+0000","last_peered":"2024-08-17T17:43:21.159273+0000","last_clean":"2024-08-17T17:43:21.159273+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:43:21.159273+0000","last_undegraded":"2024-08-17T17:43:21.159273+0000","last_fullsized":"2024-08-17T17:43:21.159273+0000","mapping_epoch":375,"log_start":"385'4300","ondisk_log_start":"385'4300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8886,"log_dups_size":0,"ondisk_log_size":8886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436736,"num_objects":2326,"num_object_clones":0,"num_object_copies":6978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13186,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":468,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928381+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:42:55.928381+0000","last_peered":"2024-08-17T17:42:55.928381+0000","last_clean":"2024-08-17T17:42:55.928381+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:42:55.928381+0000","last_undegraded":"2024-08-17T17:42:55.928381+0000","last_fullsized":"2024-08-17T17:42:55.928381+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":477,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422393+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:43:11.422393+0000","last_peered":"2024-08-17T17:43:11.422393+0000","last_clean":"2024-08-17T17:43:11.422393+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:43:11.422393+0000","last_undegraded":"2024-08-17T17:43:11.422393+0000","last_fullsized":"2024-08-17T17:43:11.422393+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":665,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.416565+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:43:11.416565+0000","last_peered":"2024-08-17T17:43:11.416565+0000","last_clean":"2024-08-17T17:43:11.416565+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:43:11.416565+0000","last_undegraded":"2024-08-17T17:43:11.416565+0000","last_fullsized":"2024-08-17T17:43:11.416565+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":712,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.418506+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:43:11.418506+0000","last_peered":"2024-08-17T17:43:11.418506+0000","last_clean":"2024-08-17T17:43:11.418506+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:43:11.418506+0000","last_undegraded":"2024-08-17T17:43:11.418506+0000","last_fullsized":"2024-08-17T17:43:11.418506+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"404'13212","reported_seq":13282,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.038871+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:43:21.038871+0000","last_peered":"2024-08-17T17:43:21.038871+0000","last_clean":"2024-08-17T17:43:21.038871+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:43:21.038871+0000","last_undegraded":"2024-08-17T17:43:21.038871+0000","last_fullsized":"2024-08-17T17:43:21.038871+0000","mapping_epoch":375,"log_start":"384'4100","ondisk_log_start":"384'4100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154402816,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13212,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":420,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.923797+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:42:55.923797+0000","last_peered":"2024-08-17T17:42:55.923797+0000","last_clean":"2024-08-17T17:42:55.923797+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:42:55.923797+0000","last_undegraded":"2024-08-17T17:42:55.923797+0000","last_fullsized":"2024-08-17T17:42:55.923797+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1120,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918294+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:42:55.918294+0000","last_peered":"2024-08-17T17:42:55.918294+0000","last_clean":"2024-08-17T17:42:55.918294+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:42:55.918294+0000","last_undegraded":"2024-08-17T17:42:55.918294+0000","last_fullsized":"2024-08-17T17:42:55.918294+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":491,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.426460+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:43:11.426460+0000","last_peered":"2024-08-17T17:43:11.426460+0000","last_clean":"2024-08-17T17:43:11.426460+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:43:11.426460+0000","last_undegraded":"2024-08-17T17:43:11.426460+0000","last_fullsized":"2024-08-17T17:43:11.426460+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917807+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:42:55.917807+0000","last_peered":"2024-08-17T17:42:55.917807+0000","last_clean":"2024-08-17T17:42:55.917807+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:42:55.917807+0000","last_undegraded":"2024-08-17T17:42:55.917807+0000","last_fullsized":"2024-08-17T17:42:55.917807+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"404'13183","reported_seq":13222,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.134241+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:43:21.134241+0000","last_peered":"2024-08-17T17:43:21.134241+0000","last_clean":"2024-08-17T17:43:21.134241+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:43:21.134241+0000","last_undegraded":"2024-08-17T17:43:21.134241+0000","last_fullsized":"2024-08-17T17:43:21.134241+0000","mapping_epoch":375,"log_start":"385'4300","ondisk_log_start":"385'4300","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8883,"log_dups_size":0,"ondisk_log_size":8883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13183,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":468,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.427260+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:43:11.427260+0000","last_peered":"2024-08-17T17:43:11.427260+0000","last_clean":"2024-08-17T17:43:11.427260+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:43:11.427260+0000","last_undegraded":"2024-08-17T17:43:11.427260+0000","last_fullsized":"2024-08-17T17:43:11.427260+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":189,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917769+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:42:55.917769+0000","last_peered":"2024-08-17T17:42:55.917769+0000","last_clean":"2024-08-17T17:42:55.917769+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:42:55.917769+0000","last_undegraded":"2024-08-17T17:42:55.917769+0000","last_fullsized":"2024-08-17T17:42:55.917769+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":958,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419131+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:43:11.419131+0000","last_peered":"2024-08-17T17:43:11.419131+0000","last_clean":"2024-08-17T17:43:11.419131+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:43:11.419131+0000","last_undegraded":"2024-08-17T17:43:11.419131+0000","last_fullsized":"2024-08-17T17:43:11.419131+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":682,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417766+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:43:11.417766+0000","last_peered":"2024-08-17T17:43:11.417766+0000","last_clean":"2024-08-17T17:43:11.417766+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:43:11.417766+0000","last_undegraded":"2024-08-17T17:43:11.417766+0000","last_fullsized":"2024-08-17T17:43:11.417766+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"404'13378","reported_seq":13417,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:20.867558+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:43:20.867558+0000","last_peered":"2024-08-17T17:43:20.867558+0000","last_clean":"2024-08-17T17:43:20.867558+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:43:20.867558+0000","last_undegraded":"2024-08-17T17:43:20.867558+0000","last_fullsized":"2024-08-17T17:43:20.867558+0000","mapping_epoch":375,"log_start":"390'6303","ondisk_log_start":"390'6303","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157155328,"num_objects":2398,"num_object_clones":0,"num_object_copies":7194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13378,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":474,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419582+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:43:11.419582+0000","last_peered":"2024-08-17T17:43:11.419582+0000","last_clean":"2024-08-17T17:43:11.419582+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:43:11.419582+0000","last_undegraded":"2024-08-17T17:43:11.419582+0000","last_fullsized":"2024-08-17T17:43:11.419582+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":679,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419042+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:43:11.419042+0000","last_peered":"2024-08-17T17:43:11.419042+0000","last_clean":"2024-08-17T17:43:11.419042+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:43:11.419042+0000","last_undegraded":"2024-08-17T17:43:11.419042+0000","last_fullsized":"2024-08-17T17:43:11.419042+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":973,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928940+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:42:55.928940+0000","last_peered":"2024-08-17T17:42:55.928940+0000","last_clean":"2024-08-17T17:42:55.928940+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:42:55.928940+0000","last_undegraded":"2024-08-17T17:42:55.928940+0000","last_fullsized":"2024-08-17T17:42:55.928940+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":676,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.416943+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:43:11.416943+0000","last_peered":"2024-08-17T17:43:11.416943+0000","last_clean":"2024-08-17T17:43:11.416943+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:43:11.416943+0000","last_undegraded":"2024-08-17T17:43:11.416943+0000","last_fullsized":"2024-08-17T17:43:11.416943+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"404'13424","reported_seq":13464,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:20.813358+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:43:20.813358+0000","last_peered":"2024-08-17T17:43:20.813358+0000","last_clean":"2024-08-17T17:43:20.813358+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:43:20.813358+0000","last_undegraded":"2024-08-17T17:43:20.813358+0000","last_fullsized":"2024-08-17T17:43:20.813358+0000","mapping_epoch":375,"log_start":"390'6414","ondisk_log_start":"390'6414","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155254808,"num_objects":2370,"num_object_clones":0,"num_object_copies":7110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2370,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13424,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":420,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918699+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:42:55.918699+0000","last_peered":"2024-08-17T17:42:55.918699+0000","last_clean":"2024-08-17T17:42:55.918699+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:42:55.918699+0000","last_undegraded":"2024-08-17T17:42:55.918699+0000","last_fullsized":"2024-08-17T17:42:55.918699+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":669,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.427563+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:43:11.427563+0000","last_peered":"2024-08-17T17:43:11.427563+0000","last_clean":"2024-08-17T17:43:11.427563+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:43:11.427563+0000","last_undegraded":"2024-08-17T17:43:11.427563+0000","last_fullsized":"2024-08-17T17:43:11.427563+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1188,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419302+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:43:11.419302+0000","last_peered":"2024-08-17T17:43:11.419302+0000","last_clean":"2024-08-17T17:43:11.419302+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:43:11.419302+0000","last_undegraded":"2024-08-17T17:43:11.419302+0000","last_fullsized":"2024-08-17T17:43:11.419302+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":667,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.423397+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:43:11.423397+0000","last_peered":"2024-08-17T17:43:11.423397+0000","last_clean":"2024-08-17T17:43:11.423397+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:43:11.423397+0000","last_undegraded":"2024-08-17T17:43:11.423397+0000","last_fullsized":"2024-08-17T17:43:11.423397+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"404'13692","reported_seq":13762,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:24.382150+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:43:24.382150+0000","last_peered":"2024-08-17T17:43:24.382150+0000","last_clean":"2024-08-17T17:43:24.382150+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:43:24.382150+0000","last_undegraded":"2024-08-17T17:43:24.382150+0000","last_fullsized":"2024-08-17T17:43:24.382150+0000","mapping_epoch":375,"log_start":"390'6700","ondisk_log_start":"390'6700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13692,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":468,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422483+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:43:11.422483+0000","last_peered":"2024-08-17T17:43:11.422483+0000","last_clean":"2024-08-17T17:43:11.422483+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:43:11.422483+0000","last_undegraded":"2024-08-17T17:43:11.422483+0000","last_fullsized":"2024-08-17T17:43:11.422483+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":688,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419736+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:43:11.419736+0000","last_peered":"2024-08-17T17:43:11.419736+0000","last_clean":"2024-08-17T17:43:11.419736+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:43:11.419736+0000","last_undegraded":"2024-08-17T17:43:11.419736+0000","last_fullsized":"2024-08-17T17:43:11.419736+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":36,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928518+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:42:55.928518+0000","last_peered":"2024-08-17T17:42:55.928518+0000","last_clean":"2024-08-17T17:42:55.928518+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:42:55.928518+0000","last_undegraded":"2024-08-17T17:42:55.928518+0000","last_fullsized":"2024-08-17T17:42:55.928518+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":348,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:43:10.689214+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:43:10.689214+0000","last_peered":"2024-08-17T17:43:10.689214+0000","last_clean":"2024-08-17T17:43:10.689214+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:43:10.689214+0000","last_undegraded":"2024-08-17T17:43:10.689214+0000","last_fullsized":"2024-08-17T17:43:10.689214+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":310,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918419+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:42:55.918419+0000","last_peered":"2024-08-17T17:42:55.918419+0000","last_clean":"2024-08-17T17:42:55.918419+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:42:55.918419+0000","last_undegraded":"2024-08-17T17:42:55.918419+0000","last_fullsized":"2024-08-17T17:42:55.918419+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":320,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918563+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:42:55.918563+0000","last_peered":"2024-08-17T17:42:55.918563+0000","last_clean":"2024-08-17T17:42:55.918563+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:42:55.918563+0000","last_undegraded":"2024-08-17T17:42:55.918563+0000","last_fullsized":"2024-08-17T17:42:55.918563+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":663,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417600+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:43:11.417600+0000","last_peered":"2024-08-17T17:43:11.417600+0000","last_clean":"2024-08-17T17:43:11.417600+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:43:11.417600+0000","last_undegraded":"2024-08-17T17:43:11.417600+0000","last_fullsized":"2024-08-17T17:43:11.417600+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":405,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.928856+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:42:55.928856+0000","last_peered":"2024-08-17T17:42:55.928856+0000","last_clean":"2024-08-17T17:42:55.928856+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:42:55.928856+0000","last_undegraded":"2024-08-17T17:42:55.928856+0000","last_fullsized":"2024-08-17T17:42:55.928856+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":189,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413363+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:43:11.413363+0000","last_peered":"2024-08-17T17:43:11.413363+0000","last_clean":"2024-08-17T17:43:11.413363+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:43:11.413363+0000","last_undegraded":"2024-08-17T17:43:11.413363+0000","last_fullsized":"2024-08-17T17:43:11.413363+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":631,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422152+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:43:11.422152+0000","last_peered":"2024-08-17T17:43:11.422152+0000","last_clean":"2024-08-17T17:43:11.422152+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:43:11.422152+0000","last_undegraded":"2024-08-17T17:43:11.422152+0000","last_fullsized":"2024-08-17T17:43:11.422152+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":689,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.420524+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:43:11.420524+0000","last_peered":"2024-08-17T17:43:11.420524+0000","last_clean":"2024-08-17T17:43:11.420524+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:43:11.420524+0000","last_undegraded":"2024-08-17T17:43:11.420524+0000","last_fullsized":"2024-08-17T17:43:11.420524+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":680,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.423315+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:43:11.423315+0000","last_peered":"2024-08-17T17:43:11.423315+0000","last_clean":"2024-08-17T17:43:11.423315+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:43:11.423315+0000","last_undegraded":"2024-08-17T17:43:11.423315+0000","last_fullsized":"2024-08-17T17:43:11.423315+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":199,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.921392+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:42:55.921392+0000","last_peered":"2024-08-17T17:42:55.921392+0000","last_clean":"2024-08-17T17:42:55.921392+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:42:55.921392+0000","last_undegraded":"2024-08-17T17:42:55.921392+0000","last_fullsized":"2024-08-17T17:42:55.921392+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":685,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.423562+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:43:11.423562+0000","last_peered":"2024-08-17T17:43:11.423562+0000","last_clean":"2024-08-17T17:43:11.423562+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:43:11.423562+0000","last_undegraded":"2024-08-17T17:43:11.423562+0000","last_fullsized":"2024-08-17T17:43:11.423562+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":663,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419921+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:43:11.419921+0000","last_peered":"2024-08-17T17:43:11.419921+0000","last_clean":"2024-08-17T17:43:11.419921+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:43:11.419921+0000","last_undegraded":"2024-08-17T17:43:11.419921+0000","last_fullsized":"2024-08-17T17:43:11.419921+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":666,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417659+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:43:11.417659+0000","last_peered":"2024-08-17T17:43:11.417659+0000","last_clean":"2024-08-17T17:43:11.417659+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:43:11.417659+0000","last_undegraded":"2024-08-17T17:43:11.417659+0000","last_fullsized":"2024-08-17T17:43:11.417659+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":681,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422287+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:43:11.422287+0000","last_peered":"2024-08-17T17:43:11.422287+0000","last_clean":"2024-08-17T17:43:11.422287+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:43:11.422287+0000","last_undegraded":"2024-08-17T17:43:11.422287+0000","last_fullsized":"2024-08-17T17:43:11.422287+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":295,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917978+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:42:55.917978+0000","last_peered":"2024-08-17T17:42:55.917978+0000","last_clean":"2024-08-17T17:42:55.917978+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:42:55.917978+0000","last_undegraded":"2024-08-17T17:42:55.917978+0000","last_fullsized":"2024-08-17T17:42:55.917978+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419268+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:43:11.419268+0000","last_peered":"2024-08-17T17:43:11.419268+0000","last_clean":"2024-08-17T17:43:11.419268+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:43:11.419268+0000","last_undegraded":"2024-08-17T17:43:11.419268+0000","last_fullsized":"2024-08-17T17:43:11.419268+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":57,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.416757+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:43:11.416757+0000","last_peered":"2024-08-17T17:43:11.416757+0000","last_clean":"2024-08-17T17:43:11.416757+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:43:11.416757+0000","last_undegraded":"2024-08-17T17:43:11.416757+0000","last_fullsized":"2024-08-17T17:43:11.416757+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":295,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.918355+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:42:55.918355+0000","last_peered":"2024-08-17T17:42:55.918355+0000","last_clean":"2024-08-17T17:42:55.918355+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:42:55.918355+0000","last_undegraded":"2024-08-17T17:42:55.918355+0000","last_fullsized":"2024-08-17T17:42:55.918355+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":192,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917964+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:42:55.917964+0000","last_peered":"2024-08-17T17:42:55.917964+0000","last_clean":"2024-08-17T17:42:55.917964+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:42:55.917964+0000","last_undegraded":"2024-08-17T17:42:55.917964+0000","last_fullsized":"2024-08-17T17:42:55.917964+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917895+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:42:55.917895+0000","last_peered":"2024-08-17T17:42:55.917895+0000","last_clean":"2024-08-17T17:42:55.917895+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:42:55.917895+0000","last_undegraded":"2024-08-17T17:42:55.917895+0000","last_fullsized":"2024-08-17T17:42:55.917895+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":202,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917905+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:42:55.917905+0000","last_peered":"2024-08-17T17:42:55.917905+0000","last_clean":"2024-08-17T17:42:55.917905+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:42:55.917905+0000","last_undegraded":"2024-08-17T17:42:55.917905+0000","last_fullsized":"2024-08-17T17:42:55.917905+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":687,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.422502+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:43:11.422502+0000","last_peered":"2024-08-17T17:43:11.422502+0000","last_clean":"2024-08-17T17:43:11.422502+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:43:11.422502+0000","last_undegraded":"2024-08-17T17:43:11.422502+0000","last_fullsized":"2024-08-17T17:43:11.422502+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":688,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.426804+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:43:11.426804+0000","last_peered":"2024-08-17T17:43:11.426804+0000","last_clean":"2024-08-17T17:43:11.426804+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:43:11.426804+0000","last_undegraded":"2024-08-17T17:43:11.426804+0000","last_fullsized":"2024-08-17T17:43:11.426804+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":310,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:42:55.917477+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:42:55.917477+0000","last_peered":"2024-08-17T17:42:55.917477+0000","last_clean":"2024-08-17T17:42:55.917477+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:42:55.917477+0000","last_undegraded":"2024-08-17T17:42:55.917477+0000","last_fullsized":"2024-08-17T17:42:55.917477+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":707,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.420429+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:43:11.420429+0000","last_peered":"2024-08-17T17:43:11.420429+0000","last_clean":"2024-08-17T17:43:11.420429+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:43:11.420429+0000","last_undegraded":"2024-08-17T17:43:11.420429+0000","last_fullsized":"2024-08-17T17:43:11.420429+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2435579928,"num_objects":37165,"num_object_clones":0,"num_object_copies":111495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4037,"num_objects_misplaced":2372,"num_objects_unfound":0,"num_objects_dirty":37165,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212821,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1390,"num_bytes_recovered":91095040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7672836096,"data_stored":7672823880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126466,"ondisk_log_size":126466,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201758,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2064656,"kb_used_data":1245128,"kb_used_omap":1,"kb_used_meta":819454,"kb_avail":91680496,"statfs":{"total":95995035648,"available":93880827904,"internally_reserved":0,"allocated":1275011072,"data_stored":1273021336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":839121355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922255,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":820604,"kb_used_data":426164,"kb_used_omap":7,"kb_used_meta":394424,"kb_avail":92924548,"statfs":{"total":95995035648,"available":95154737152,"internally_reserved":0,"allocated":436391936,"data_stored":434421622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":403890870},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085778,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1854468,"kb_used_data":1388412,"kb_used_omap":7,"kb_used_meta":466040,"kb_avail":91890684,"statfs":{"total":95995035648,"available":94096060416,"internally_reserved":0,"allocated":1421733888,"data_stored":1419752683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":477225401},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986668,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1203464,"kb_used_data":478336,"kb_used_omap":1,"kb_used_meta":725118,"kb_avail":92541688,"statfs":{"total":95995035648,"available":94762688512,"internally_reserved":0,"allocated":489816064,"data_stored":487834434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":742521286},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435560,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055672,"kb_used_data":1135472,"kb_used_omap":5,"kb_used_meta":920186,"kb_avail":91689480,"statfs":{"total":95995035648,"available":93890027520,"internally_reserved":0,"allocated":1162723328,"data_stored":1160753826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":942270537},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599086,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263608,"kb_used_data":460016,"kb_used_omap":7,"kb_used_meta":803576,"kb_avail":92481544,"statfs":{"total":95995035648,"available":94701101056,"internally_reserved":0,"allocated":471056384,"data_stored":469086231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":822861997},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762612,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2651184,"kb_used_data":1857320,"kb_used_omap":7,"kb_used_meta":793848,"kb_avail":91093968,"statfs":{"total":95995035648,"available":93280223232,"internally_reserved":0,"allocated":1901895680,"data_stored":1899932329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":812900769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926138,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1474620,"kb_used_data":630772,"kb_used_omap":7,"kb_used_meta":843832,"kb_avail":92270532,"statfs":{"total":95995035648,"available":94485024768,"internally_reserved":0,"allocated":645910528,"data_stored":643937173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":864084572},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":623247360,"data_stored":623247360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1889144832,"data_stored":1889140760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":460390400,"data_stored":460390400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1149763584,"data_stored":1149763584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":479002624,"data_stored":479002624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1392185344,"data_stored":1392181272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":431226880,"data_stored":431226880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1247875072,"data_stored":1247871000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:26.691 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:26.792 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:26.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:26.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:26.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:27.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.099 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:27.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:26 smithi175 ceph-mon[101434]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 4037/111579 objects degraded (3.618%); 2372/111579 objects misplaced (2.126%); 778 KiB/s, 12 objects/s recovering 2024-08-17T17:43:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:26 smithi175 ceph-mon[101434]: from='client.35964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:27.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:26 smithi064 ceph-mon[104516]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 4037/111579 objects degraded (3.618%); 2372/111579 objects misplaced (2.126%); 778 KiB/s, 12 objects/s recovering 2024-08-17T17:43:27.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:26 smithi064 ceph-mon[104516]: from='client.35964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:27.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:26 smithi064 ceph-mon[109246]: pgmap v1277: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 4037/111579 objects degraded (3.618%); 2372/111579 objects misplaced (2.126%); 778 KiB/s, 12 objects/s recovering 2024-08-17T17:43:27.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:26 smithi064 ceph-mon[109246]: from='client.35964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:27.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:27.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:27.339 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:27.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:27.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:27.514 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:27.616 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:27.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:27.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:27.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:27.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:27.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:27.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:27.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:28.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:28.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:28.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:28.205 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:28.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:28.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:28.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:28.484 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:28.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:28.625 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:28.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:28.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:28.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:28.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:28.916 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:29.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:29.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:29.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:28 smithi175 ceph-mon[101434]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:43:29.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:28 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4240/102258 objects degraded (4.146%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:28 smithi064 ceph-mon[104516]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:43:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:28 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4240/102258 objects degraded (4.146%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:28 smithi064 ceph-mon[109246]: pgmap v1278: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:43:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:28 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4240/102258 objects degraded (4.146%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:29.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:29.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:29.336 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:29.438 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:29.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:29.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:29.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:29.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:29.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:29.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:29.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:29.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.879 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:29.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:29.927 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:43:29.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:30.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:30.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:30.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:30.153 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:30.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[104516]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:43:30.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3817074529' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[109246]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:43:30.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3817074529' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:30.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:30.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:29 smithi175 ceph-mon[101434]: pgmap v1279: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4240/102258 objects degraded (4.146%); 2102/102258 objects misplaced (2.056%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:43:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3817074529' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:30.443 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:30.545 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:30.608 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:30.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:30.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:30.742 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:30.844 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:30.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:30.978 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:31.007 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:31.109 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:31.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:31.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:31.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[109246]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:43:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:30 smithi064 ceph-mon[104516]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:43:31.248 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:31.277 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:31.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:31.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:30 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:31.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:31.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:30 smithi175 ceph-mon[101434]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:43:31.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.419 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:31.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:31.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:31.522 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:43:31.522 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:31.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:31.573 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:43:31.573 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:31.620 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:43:31.620 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:31.655 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:43:31.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:31.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:31.691 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:43:31.691 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:31.702 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:31.722 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:43:31.723 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:31.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:31.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:31.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:43:31.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:31.788 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:43:31.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:31.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.881 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:31.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:31.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:31.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:31 smithi064 ceph-mon[104516]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4456/91458 objects degraded (4.872%); 1817/91458 objects misplaced (1.987%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:31.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/294199587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:31.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:31 smithi064 ceph-mon[109246]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4456/91458 objects degraded (4.872%); 1817/91458 objects misplaced (1.987%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:31.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/294199587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.030 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:32.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:32.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:32.169 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:32.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:32.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:32.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:31 smithi175 ceph-mon[101434]: pgmap v1281: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4456/91458 objects degraded (4.872%); 1817/91458 objects misplaced (1.987%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:32.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/294199587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:32.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:32.450 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:32.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:32.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:32.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:32.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:32.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:32.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:32.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:33.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:33.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:43:33.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:33.126 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:33.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:33.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:33.329 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:33.330 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:33.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:33.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:33.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:33.558 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:33.660 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:33.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:33.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.842 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:33.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:33.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:33.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:34.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:34.116 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:34.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:34.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:34.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:34.379 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:33] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T17:43:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:34.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:34.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.538 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:34.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:34.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:34 smithi064 ceph-mon[104516]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4456/89847 objects degraded (4.960%); 1817/89847 objects misplaced (2.022%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:34 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4456/89847 objects degraded (4.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:34 smithi064 ceph-mon[109246]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4456/89847 objects degraded (4.960%); 1817/89847 objects misplaced (2.022%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:34 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4456/89847 objects degraded (4.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:34.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:34 smithi175 ceph-mon[101434]: pgmap v1282: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4456/89847 objects degraded (4.960%); 1817/89847 objects misplaced (2.022%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:34.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:34 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4456/89847 objects degraded (4.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:34.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:34.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:34.809 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:34.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:34.943 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:35.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:35.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:35.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:35.216 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:35.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:35.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:35.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:35.482 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:35.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:35.616 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:35.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:35.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:35.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:43:35.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:35.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:35.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:36.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:36.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.165 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:36.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:36.266 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:36.302 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:36.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:36.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:36.539 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:36.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:36.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:36 smithi175 ceph-mon[101434]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 4456/86853 objects degraded (5.131%); 1817/86853 objects misplaced (2.092%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:36.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:36.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:36 smithi064 ceph-mon[104516]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 4456/86853 objects degraded (5.131%); 1817/86853 objects misplaced (2.092%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:36.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:36 smithi064 ceph-mon[109246]: pgmap v1283: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 4456/86853 objects degraded (5.131%); 1817/86853 objects misplaced (2.092%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:43:36.698 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:36.800 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:36.834 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:36.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:36.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:37.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:37.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:37.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:43:37.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:37.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:37.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.366 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:37.467 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:37.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:37.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:37.632 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:37.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:37.757 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:37.859 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:37.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:37.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:38.009 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:38.111 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:38.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:38.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:38.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:38.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:38.413 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:38.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:38.548 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:38.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:38.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:38 smithi175 ceph-mon[101434]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:38 smithi175 ceph-mon[101434]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:38.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:38 smithi064 ceph-mon[104516]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:38 smithi064 ceph-mon[104516]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:38 smithi064 ceph-mon[109246]: pgmap v1284: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:38 smithi064 ceph-mon[109246]: from='client.35979 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:38.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:38.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:38.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.957 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:38.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:39.057 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:39.085 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:39.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:39.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.233 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:39.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:39.334 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:39.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:39.449 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:43:39.454 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:43:39.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:39.502 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:39.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:39.604 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.640 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:39.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:39.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:39 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4615/77652 objects degraded (5.943%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:39.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:39 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4615/77652 objects degraded (5.943%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:39.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:39 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4615/77652 objects degraded (5.943%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:39.741 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:39.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:39.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:39.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:39.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:39.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:40.037 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:40.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:40.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:40.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:40.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:40.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:40.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:40.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:40.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:40.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:40.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:40 smithi064 ceph-mon[109246]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:40.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:40 smithi064 ceph-mon[104516]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:40.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:40.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:40.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:40.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:40 smithi175 ceph-mon[101434]: pgmap v1285: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4615/77652 objects degraded (5.943%); 1575/77652 objects misplaced (2.028%); 972 KiB/s, 15 objects/s recovering 2024-08-17T17:43:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:40.956 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.993 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:40.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:41.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:41.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:41.243 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.275 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:41.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:41.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:41.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:41.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:41.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:41.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:41.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:41.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:41.815 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:41.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:41.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:42.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:42.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:42.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:42.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:42.354 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:42.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:42.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:42.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:42.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:42.617 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:42 smithi064 ceph-mon[104516]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4721/67110 objects degraded (7.035%); 1362/67110 objects misplaced (2.030%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:43:42.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:42 smithi064 ceph-mon[109246]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4721/67110 objects degraded (7.035%); 1362/67110 objects misplaced (2.030%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:43:42.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:42.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:42.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:42.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:42.881 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:42 smithi175 ceph-mon[101434]: pgmap v1286: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4721/67110 objects degraded (7.035%); 1362/67110 objects misplaced (2.030%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:43:42.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:42.996 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1284,"stamp":"2024-08-17T17:43:37.321827+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1699223549,"num_objects":25884,"num_object_clones":0,"num_object_copies":77652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4615,"num_objects_misplaced":1575,"num_objects_unfound":0,"num_objects_dirty":25884,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":225038,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1730,"num_bytes_recovered":113842403,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129147,"ondisk_log_size":129147,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11266452,"kb_used_data":5488404,"kb_used_omap":46,"kb_used_meta":5777873,"kb_avail":738694764,"statfs":{"total":767960285184,"available":756423438336,"internally_reserved":0,"allocated":5620125696,"data_stored":5604295154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5916542195},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-537526272,"num_objects":-8202,"num_object_clones":0,"num_object_copies":-24606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":375,"num_objects_misplaced":-527,"num_objects_unfound":0,"num_objects_dirty":-8202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8202,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":152,"num_bytes_recovered":9961472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006399"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":671,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.934279+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:43:29.934279+0000","last_peered":"2024-08-17T17:43:29.934279+0000","last_clean":"2024-08-17T17:43:29.934279+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:43:29.934279+0000","last_undegraded":"2024-08-17T17:43:29.934279+0000","last_fullsized":"2024-08-17T17:43:29.934279+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":406,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419228+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:43:11.419228+0000","last_peered":"2024-08-17T17:43:11.419228+0000","last_clean":"2024-08-17T17:43:11.419228+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:43:11.419228+0000","last_undegraded":"2024-08-17T17:43:11.419228+0000","last_fullsized":"2024-08-17T17:43:11.419228+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.225880+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:43:21.225880+0000","last_peered":"2024-08-17T17:43:21.225880+0000","last_clean":"2024-08-17T17:43:21.225880+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:43:21.225880+0000","last_undegraded":"2024-08-17T17:43:21.225880+0000","last_fullsized":"2024-08-17T17:43:21.225880+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":663,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952262+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:43:29.952262+0000","last_peered":"2024-08-17T17:43:29.952262+0000","last_clean":"2024-08-17T17:43:29.952262+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:43:29.952262+0000","last_undegraded":"2024-08-17T17:43:29.952262+0000","last_fullsized":"2024-08-17T17:43:29.952262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":662,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952670+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:43:29.952670+0000","last_peered":"2024-08-17T17:43:29.952670+0000","last_clean":"2024-08-17T17:43:29.952670+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:43:29.952670+0000","last_undegraded":"2024-08-17T17:43:29.952670+0000","last_fullsized":"2024-08-17T17:43:29.952670+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":392,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419716+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:43:11.419716+0000","last_peered":"2024-08-17T17:43:11.419716+0000","last_clean":"2024-08-17T17:43:11.419716+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:43:11.419716+0000","last_undegraded":"2024-08-17T17:43:11.419716+0000","last_fullsized":"2024-08-17T17:43:11.419716+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":671,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928564+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:43:29.928564+0000","last_peered":"2024-08-17T17:43:29.928564+0000","last_clean":"2024-08-17T17:43:29.928564+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:43:29.928564+0000","last_undegraded":"2024-08-17T17:43:29.928564+0000","last_fullsized":"2024-08-17T17:43:29.928564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":340,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:14.644932+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:43:14.644932+0000","last_peered":"2024-08-17T17:43:14.644932+0000","last_clean":"2024-08-17T17:43:14.644932+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:43:14.644932+0000","last_undegraded":"2024-08-17T17:43:14.644932+0000","last_fullsized":"2024-08-17T17:43:14.644932+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"405'14046","reported_seq":17335,"reported_epoch":405,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:36.146362+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:43:36.146362+0000","last_peered":"2024-08-17T17:43:36.146362+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:43:36.146362+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"385'4900","ondisk_log_start":"385'4900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9146,"log_dups_size":0,"ondisk_log_size":9146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4615,"num_objects_misplaced":1575,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14046,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1609,"num_bytes_recovered":105447424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":1575},{"shards":"0,5","objects":-4592}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":401,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419636+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:43:11.419636+0000","last_peered":"2024-08-17T17:43:11.419636+0000","last_clean":"2024-08-17T17:43:11.419636+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:43:11.419636+0000","last_undegraded":"2024-08-17T17:43:11.419636+0000","last_fullsized":"2024-08-17T17:43:11.419636+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":200,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413127+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:43:11.413127+0000","last_peered":"2024-08-17T17:43:11.413127+0000","last_clean":"2024-08-17T17:43:11.413127+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:43:11.413127+0000","last_undegraded":"2024-08-17T17:43:11.413127+0000","last_fullsized":"2024-08-17T17:43:11.413127+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"405'14016","reported_seq":14088,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.409086+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:43:36.409086+0000","last_peered":"2024-08-17T17:43:36.409086+0000","last_clean":"2024-08-17T17:43:36.409086+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:43:36.409086+0000","last_undegraded":"2024-08-17T17:43:36.409086+0000","last_fullsized":"2024-08-17T17:43:36.409086+0000","mapping_epoch":375,"log_start":"390'6800","ondisk_log_start":"390'6800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7216,"log_dups_size":0,"ondisk_log_size":7216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106430464,"num_objects":1624,"num_object_clones":0,"num_object_copies":4872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14016,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":402,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419363+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:43:11.419363+0000","last_peered":"2024-08-17T17:43:11.419363+0000","last_clean":"2024-08-17T17:43:11.419363+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:43:11.419363+0000","last_undegraded":"2024-08-17T17:43:11.419363+0000","last_fullsized":"2024-08-17T17:43:11.419363+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":297,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407422+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:43:11.407422+0000","last_peered":"2024-08-17T17:43:11.407422+0000","last_clean":"2024-08-17T17:43:11.407422+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:43:11.407422+0000","last_undegraded":"2024-08-17T17:43:11.407422+0000","last_fullsized":"2024-08-17T17:43:11.407422+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"405'14028","reported_seq":14067,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.233177+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:43:36.233177+0000","last_peered":"2024-08-17T17:43:36.233177+0000","last_clean":"2024-08-17T17:43:36.233177+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:43:36.233177+0000","last_undegraded":"2024-08-17T17:43:36.233177+0000","last_fullsized":"2024-08-17T17:43:36.233177+0000","mapping_epoch":375,"log_start":"387'5200","ondisk_log_start":"387'5200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105381888,"num_objects":1608,"num_object_clones":0,"num_object_copies":4824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14028,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":419,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.430412+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:43:11.430412+0000","last_peered":"2024-08-17T17:43:11.430412+0000","last_clean":"2024-08-17T17:43:11.430412+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:43:11.430412+0000","last_undegraded":"2024-08-17T17:43:11.430412+0000","last_fullsized":"2024-08-17T17:43:11.430412+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":320,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.438044+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:43:11.438044+0000","last_peered":"2024-08-17T17:43:11.438044+0000","last_clean":"2024-08-17T17:43:11.438044+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:43:11.438044+0000","last_undegraded":"2024-08-17T17:43:11.438044+0000","last_fullsized":"2024-08-17T17:43:11.438044+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"405'14066","reported_seq":14106,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.203774+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:43:36.203774+0000","last_peered":"2024-08-17T17:43:36.203774+0000","last_clean":"2024-08-17T17:43:36.203774+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:43:36.203774+0000","last_undegraded":"2024-08-17T17:43:36.203774+0000","last_fullsized":"2024-08-17T17:43:36.203774+0000","mapping_epoch":375,"log_start":"387'5200","ondisk_log_start":"387'5200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14066,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":200,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413437+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:43:11.413437+0000","last_peered":"2024-08-17T17:43:11.413437+0000","last_clean":"2024-08-17T17:43:11.413437+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:43:11.413437+0000","last_undegraded":"2024-08-17T17:43:11.413437+0000","last_fullsized":"2024-08-17T17:43:11.413437+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":677,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.958738+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:43:29.958738+0000","last_peered":"2024-08-17T17:43:29.958738+0000","last_clean":"2024-08-17T17:43:29.958738+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:43:29.958738+0000","last_undegraded":"2024-08-17T17:43:29.958738+0000","last_fullsized":"2024-08-17T17:43:29.958738+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"405'14055","reported_seq":14127,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:34.414336+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:43:34.414336+0000","last_peered":"2024-08-17T17:43:34.414336+0000","last_clean":"2024-08-17T17:43:34.414336+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:43:34.414336+0000","last_undegraded":"2024-08-17T17:43:34.414336+0000","last_fullsized":"2024-08-17T17:43:34.414336+0000","mapping_epoch":375,"log_start":"391'7000","ondisk_log_start":"391'7000","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132096,"num_objects":1711,"num_object_clones":0,"num_object_copies":5133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14055,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":760,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419598+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:43:11.419598+0000","last_peered":"2024-08-17T17:43:11.419598+0000","last_clean":"2024-08-17T17:43:11.419598+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:43:11.419598+0000","last_undegraded":"2024-08-17T17:43:11.419598+0000","last_fullsized":"2024-08-17T17:43:11.419598+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":692,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928372+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:43:29.928372+0000","last_peered":"2024-08-17T17:43:29.928372+0000","last_clean":"2024-08-17T17:43:29.928372+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:43:29.928372+0000","last_undegraded":"2024-08-17T17:43:29.928372+0000","last_fullsized":"2024-08-17T17:43:29.928372+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":321,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.408312+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:43:11.408312+0000","last_peered":"2024-08-17T17:43:11.408312+0000","last_clean":"2024-08-17T17:43:11.408312+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:43:11.408312+0000","last_undegraded":"2024-08-17T17:43:11.408312+0000","last_fullsized":"2024-08-17T17:43:11.408312+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"405'13704","reported_seq":13776,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.465206+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:43:36.465206+0000","last_peered":"2024-08-17T17:43:36.465206+0000","last_clean":"2024-08-17T17:43:36.465206+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:43:36.465206+0000","last_undegraded":"2024-08-17T17:43:36.465206+0000","last_fullsized":"2024-08-17T17:43:36.465206+0000","mapping_epoch":375,"log_start":"390'6500","ondisk_log_start":"390'6500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100532224,"num_objects":1534,"num_object_clones":0,"num_object_copies":4602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13704,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":421,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413050+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:43:11.413050+0000","last_peered":"2024-08-17T17:43:11.413050+0000","last_clean":"2024-08-17T17:43:11.413050+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:43:11.413050+0000","last_undegraded":"2024-08-17T17:43:11.413050+0000","last_fullsized":"2024-08-17T17:43:11.413050+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":327,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407600+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:43:11.407600+0000","last_peered":"2024-08-17T17:43:11.407600+0000","last_clean":"2024-08-17T17:43:11.407600+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:43:11.407600+0000","last_undegraded":"2024-08-17T17:43:11.407600+0000","last_fullsized":"2024-08-17T17:43:11.407600+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":297,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.408226+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:43:11.408226+0000","last_peered":"2024-08-17T17:43:11.408226+0000","last_clean":"2024-08-17T17:43:11.408226+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:43:11.408226+0000","last_undegraded":"2024-08-17T17:43:11.408226+0000","last_fullsized":"2024-08-17T17:43:11.408226+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"405'13865","reported_seq":13905,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:35.867960+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:43:35.867960+0000","last_peered":"2024-08-17T17:43:35.867960+0000","last_clean":"2024-08-17T17:43:35.867960+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:43:35.867960+0000","last_undegraded":"2024-08-17T17:43:35.867960+0000","last_fullsized":"2024-08-17T17:43:35.867960+0000","mapping_epoch":375,"log_start":"390'6838","ondisk_log_start":"390'6838","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7027,"log_dups_size":0,"ondisk_log_size":7027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103481344,"num_objects":1579,"num_object_clones":0,"num_object_copies":4737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13865,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":297,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407317+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:43:11.407317+0000","last_peered":"2024-08-17T17:43:11.407317+0000","last_clean":"2024-08-17T17:43:11.407317+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:43:11.407317+0000","last_undegraded":"2024-08-17T17:43:11.407317+0000","last_fullsized":"2024-08-17T17:43:11.407317+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":312,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.408505+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:43:11.408505+0000","last_peered":"2024-08-17T17:43:11.408505+0000","last_clean":"2024-08-17T17:43:11.408505+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:43:11.408505+0000","last_undegraded":"2024-08-17T17:43:11.408505+0000","last_fullsized":"2024-08-17T17:43:11.408505+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"405'13703","reported_seq":13775,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:34.406445+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:43:34.406445+0000","last_peered":"2024-08-17T17:43:34.406445+0000","last_clean":"2024-08-17T17:43:34.406445+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:43:34.406445+0000","last_undegraded":"2024-08-17T17:43:34.406445+0000","last_fullsized":"2024-08-17T17:43:34.406445+0000","mapping_epoch":375,"log_start":"390'6700","ondisk_log_start":"390'6700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109641728,"num_objects":1673,"num_object_clones":0,"num_object_copies":5019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13703,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":421,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413666+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:43:11.413666+0000","last_peered":"2024-08-17T17:43:11.413666+0000","last_clean":"2024-08-17T17:43:11.413666+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:43:11.413666+0000","last_undegraded":"2024-08-17T17:43:11.413666+0000","last_fullsized":"2024-08-17T17:43:11.413666+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":673,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.930978+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:43:29.930978+0000","last_peered":"2024-08-17T17:43:29.930978+0000","last_clean":"2024-08-17T17:43:29.930978+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:43:29.930978+0000","last_undegraded":"2024-08-17T17:43:29.930978+0000","last_fullsized":"2024-08-17T17:43:29.930978+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":661,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.934027+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:43:29.934027+0000","last_peered":"2024-08-17T17:43:29.934027+0000","last_clean":"2024-08-17T17:43:29.934027+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:43:29.934027+0000","last_undegraded":"2024-08-17T17:43:29.934027+0000","last_fullsized":"2024-08-17T17:43:29.934027+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"405'14114","reported_seq":14186,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:34.425948+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:43:34.425948+0000","last_peered":"2024-08-17T17:43:34.425948+0000","last_clean":"2024-08-17T17:43:34.425948+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:43:34.425948+0000","last_undegraded":"2024-08-17T17:43:34.425948+0000","last_fullsized":"2024-08-17T17:43:34.425948+0000","mapping_epoch":375,"log_start":"391'7100","ondisk_log_start":"391'7100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110493696,"num_objects":1686,"num_object_clones":0,"num_object_copies":5058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14114,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":744,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.418982+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:43:11.418982+0000","last_peered":"2024-08-17T17:43:11.418982+0000","last_clean":"2024-08-17T17:43:11.418982+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:43:11.418982+0000","last_undegraded":"2024-08-17T17:43:11.418982+0000","last_fullsized":"2024-08-17T17:43:11.418982+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":998,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413766+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:43:11.413766+0000","last_peered":"2024-08-17T17:43:11.413766+0000","last_clean":"2024-08-17T17:43:11.413766+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:43:11.413766+0000","last_undegraded":"2024-08-17T17:43:11.413766+0000","last_fullsized":"2024-08-17T17:43:11.413766+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":656,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928394+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:43:29.928394+0000","last_peered":"2024-08-17T17:43:29.928394+0000","last_clean":"2024-08-17T17:43:29.928394+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:43:29.928394+0000","last_undegraded":"2024-08-17T17:43:29.928394+0000","last_fullsized":"2024-08-17T17:43:29.928394+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":311,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.408145+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:43:11.408145+0000","last_peered":"2024-08-17T17:43:11.408145+0000","last_clean":"2024-08-17T17:43:11.408145+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:43:11.408145+0000","last_undegraded":"2024-08-17T17:43:11.408145+0000","last_fullsized":"2024-08-17T17:43:11.408145+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"405'14033","reported_seq":14105,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.126483+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:43:36.126483+0000","last_peered":"2024-08-17T17:43:36.126483+0000","last_clean":"2024-08-17T17:43:36.126483+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:43:36.126483+0000","last_undegraded":"2024-08-17T17:43:36.126483+0000","last_fullsized":"2024-08-17T17:43:36.126483+0000","mapping_epoch":375,"log_start":"385'4900","ondisk_log_start":"385'4900","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9133,"log_dups_size":0,"ondisk_log_size":9133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106496000,"num_objects":1625,"num_object_clones":0,"num_object_copies":4875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14033,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":473,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.943359+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:43:29.943359+0000","last_peered":"2024-08-17T17:43:29.943359+0000","last_clean":"2024-08-17T17:43:29.943359+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:43:29.943359+0000","last_undegraded":"2024-08-17T17:43:29.943359+0000","last_fullsized":"2024-08-17T17:43:29.943359+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":785,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.418931+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:43:11.418931+0000","last_peered":"2024-08-17T17:43:11.418931+0000","last_clean":"2024-08-17T17:43:11.418931+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:43:11.418931+0000","last_undegraded":"2024-08-17T17:43:11.418931+0000","last_fullsized":"2024-08-17T17:43:11.418931+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":678,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.962787+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:43:29.962787+0000","last_peered":"2024-08-17T17:43:29.962787+0000","last_clean":"2024-08-17T17:43:29.962787+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:43:29.962787+0000","last_undegraded":"2024-08-17T17:43:29.962787+0000","last_fullsized":"2024-08-17T17:43:29.962787+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":675,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952817+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:43:29.952817+0000","last_peered":"2024-08-17T17:43:29.952817+0000","last_clean":"2024-08-17T17:43:29.952817+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:43:29.952817+0000","last_undegraded":"2024-08-17T17:43:29.952817+0000","last_fullsized":"2024-08-17T17:43:29.952817+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"405'13983","reported_seq":14023,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.203735+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:43:36.203735+0000","last_peered":"2024-08-17T17:43:36.203735+0000","last_clean":"2024-08-17T17:43:36.203735+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:43:36.203735+0000","last_undegraded":"2024-08-17T17:43:36.203735+0000","last_fullsized":"2024-08-17T17:43:36.203735+0000","mapping_epoch":375,"log_start":"387'5100","ondisk_log_start":"387'5100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8883,"log_dups_size":0,"ondisk_log_size":8883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13983,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":470,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935618+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:43:29.935618+0000","last_peered":"2024-08-17T17:43:29.935618+0000","last_clean":"2024-08-17T17:43:29.935618+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:43:29.935618+0000","last_undegraded":"2024-08-17T17:43:29.935618+0000","last_fullsized":"2024-08-17T17:43:29.935618+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":479,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928029+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:43:29.928029+0000","last_peered":"2024-08-17T17:43:29.928029+0000","last_clean":"2024-08-17T17:43:29.928029+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:43:29.928029+0000","last_undegraded":"2024-08-17T17:43:29.928029+0000","last_fullsized":"2024-08-17T17:43:29.928029+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":667,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.957745+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:43:29.957745+0000","last_peered":"2024-08-17T17:43:29.957745+0000","last_clean":"2024-08-17T17:43:29.957745+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:43:29.957745+0000","last_undegraded":"2024-08-17T17:43:29.957745+0000","last_fullsized":"2024-08-17T17:43:29.957745+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":714,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.944631+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:43:29.944631+0000","last_peered":"2024-08-17T17:43:29.944631+0000","last_clean":"2024-08-17T17:43:29.944631+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:43:29.944631+0000","last_undegraded":"2024-08-17T17:43:29.944631+0000","last_fullsized":"2024-08-17T17:43:29.944631+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"405'13970","reported_seq":14042,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.092729+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:43:36.092729+0000","last_peered":"2024-08-17T17:43:36.092729+0000","last_clean":"2024-08-17T17:43:36.092729+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:43:36.092729+0000","last_undegraded":"2024-08-17T17:43:36.092729+0000","last_fullsized":"2024-08-17T17:43:36.092729+0000","mapping_epoch":375,"log_start":"385'4800","ondisk_log_start":"385'4800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9170,"log_dups_size":0,"ondisk_log_size":9170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13970,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":421,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413677+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:43:11.413677+0000","last_peered":"2024-08-17T17:43:11.413677+0000","last_clean":"2024-08-17T17:43:11.413677+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:43:11.413677+0000","last_undegraded":"2024-08-17T17:43:11.413677+0000","last_fullsized":"2024-08-17T17:43:11.413677+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1121,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413837+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:43:11.413837+0000","last_peered":"2024-08-17T17:43:11.413837+0000","last_clean":"2024-08-17T17:43:11.413837+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:43:11.413837+0000","last_undegraded":"2024-08-17T17:43:11.413837+0000","last_fullsized":"2024-08-17T17:43:11.413837+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":493,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.933235+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:43:29.933235+0000","last_peered":"2024-08-17T17:43:29.933235+0000","last_clean":"2024-08-17T17:43:29.933235+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:43:29.933235+0000","last_undegraded":"2024-08-17T17:43:29.933235+0000","last_fullsized":"2024-08-17T17:43:29.933235+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":53,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.933059+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:43:29.933059+0000","last_peered":"2024-08-17T17:43:29.933059+0000","last_clean":"2024-08-17T17:43:29.933059+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:43:29.933059+0000","last_undegraded":"2024-08-17T17:43:29.933059+0000","last_fullsized":"2024-08-17T17:43:29.933059+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"405'13938","reported_seq":13978,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.174581+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:43:36.174581+0000","last_peered":"2024-08-17T17:43:36.174581+0000","last_clean":"2024-08-17T17:43:36.174581+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:43:36.174581+0000","last_undegraded":"2024-08-17T17:43:36.174581+0000","last_fullsized":"2024-08-17T17:43:36.174581+0000","mapping_epoch":375,"log_start":"387'5100","ondisk_log_start":"387'5100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102891520,"num_objects":1570,"num_object_clones":0,"num_object_copies":4710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13938,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":470,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928458+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:43:29.928458+0000","last_peered":"2024-08-17T17:43:29.928458+0000","last_clean":"2024-08-17T17:43:29.928458+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:43:29.928458+0000","last_undegraded":"2024-08-17T17:43:29.928458+0000","last_fullsized":"2024-08-17T17:43:29.928458+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417730+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:43:11.417730+0000","last_peered":"2024-08-17T17:43:11.417730+0000","last_clean":"2024-08-17T17:43:11.417730+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:43:11.417730+0000","last_undegraded":"2024-08-17T17:43:11.417730+0000","last_fullsized":"2024-08-17T17:43:11.417730+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":960,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952147+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:43:29.952147+0000","last_peered":"2024-08-17T17:43:29.952147+0000","last_clean":"2024-08-17T17:43:29.952147+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:43:29.952147+0000","last_undegraded":"2024-08-17T17:43:29.952147+0000","last_fullsized":"2024-08-17T17:43:29.952147+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":684,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.949577+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:43:29.949577+0000","last_peered":"2024-08-17T17:43:29.949577+0000","last_clean":"2024-08-17T17:43:29.949577+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:43:29.949577+0000","last_undegraded":"2024-08-17T17:43:29.949577+0000","last_fullsized":"2024-08-17T17:43:29.949577+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"405'14179","reported_seq":14218,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:35.903715+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:43:35.903715+0000","last_peered":"2024-08-17T17:43:35.903715+0000","last_clean":"2024-08-17T17:43:35.903715+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:43:35.903715+0000","last_undegraded":"2024-08-17T17:43:35.903715+0000","last_fullsized":"2024-08-17T17:43:35.903715+0000","mapping_epoch":375,"log_start":"391'7103","ondisk_log_start":"391'7103","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14179,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":476,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952492+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:43:29.952492+0000","last_peered":"2024-08-17T17:43:29.952492+0000","last_clean":"2024-08-17T17:43:29.952492+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:43:29.952492+0000","last_undegraded":"2024-08-17T17:43:29.952492+0000","last_fullsized":"2024-08-17T17:43:29.952492+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":681,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952048+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:43:29.952048+0000","last_peered":"2024-08-17T17:43:29.952048+0000","last_clean":"2024-08-17T17:43:29.952048+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:43:29.952048+0000","last_undegraded":"2024-08-17T17:43:29.952048+0000","last_fullsized":"2024-08-17T17:43:29.952048+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":974,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.430402+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:43:11.430402+0000","last_peered":"2024-08-17T17:43:11.430402+0000","last_clean":"2024-08-17T17:43:11.430402+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:43:11.430402+0000","last_undegraded":"2024-08-17T17:43:11.430402+0000","last_fullsized":"2024-08-17T17:43:11.430402+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":678,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.956077+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:43:29.956077+0000","last_peered":"2024-08-17T17:43:29.956077+0000","last_clean":"2024-08-17T17:43:29.956077+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:43:29.956077+0000","last_undegraded":"2024-08-17T17:43:29.956077+0000","last_fullsized":"2024-08-17T17:43:29.956077+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"405'14209","reported_seq":14250,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:35.840206+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:43:35.840206+0000","last_peered":"2024-08-17T17:43:35.840206+0000","last_clean":"2024-08-17T17:43:35.840206+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:43:35.840206+0000","last_undegraded":"2024-08-17T17:43:35.840206+0000","last_fullsized":"2024-08-17T17:43:35.840206+0000","mapping_epoch":375,"log_start":"391'7214","ondisk_log_start":"391'7214","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103809048,"num_objects":1585,"num_object_clones":0,"num_object_copies":4755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14209,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":421,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.408071+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:43:11.408071+0000","last_peered":"2024-08-17T17:43:11.408071+0000","last_clean":"2024-08-17T17:43:11.408071+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:43:11.408071+0000","last_undegraded":"2024-08-17T17:43:11.408071+0000","last_fullsized":"2024-08-17T17:43:11.408071+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":671,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928756+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:43:29.928756+0000","last_peered":"2024-08-17T17:43:29.928756+0000","last_clean":"2024-08-17T17:43:29.928756+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:43:29.928756+0000","last_undegraded":"2024-08-17T17:43:29.928756+0000","last_fullsized":"2024-08-17T17:43:29.928756+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1190,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.950989+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:43:29.950989+0000","last_peered":"2024-08-17T17:43:29.950989+0000","last_clean":"2024-08-17T17:43:29.950989+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:43:29.950989+0000","last_undegraded":"2024-08-17T17:43:29.950989+0000","last_fullsized":"2024-08-17T17:43:29.950989+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":669,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.930661+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:43:29.930661+0000","last_peered":"2024-08-17T17:43:29.930661+0000","last_clean":"2024-08-17T17:43:29.930661+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:43:29.930661+0000","last_undegraded":"2024-08-17T17:43:29.930661+0000","last_fullsized":"2024-08-17T17:43:29.930661+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"405'14221","reported_seq":14293,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:34.395842+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:43:34.395842+0000","last_peered":"2024-08-17T17:43:34.395842+0000","last_clean":"2024-08-17T17:43:34.395842+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:43:34.395842+0000","last_undegraded":"2024-08-17T17:43:34.395842+0000","last_fullsized":"2024-08-17T17:43:34.395842+0000","mapping_epoch":375,"log_start":"391'7200","ondisk_log_start":"391'7200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113704960,"num_objects":1735,"num_object_clones":0,"num_object_copies":5205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14221,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":470,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.934541+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:43:29.934541+0000","last_peered":"2024-08-17T17:43:29.934541+0000","last_clean":"2024-08-17T17:43:29.934541+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:43:29.934541+0000","last_undegraded":"2024-08-17T17:43:29.934541+0000","last_fullsized":"2024-08-17T17:43:29.934541+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":690,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952581+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:43:29.952581+0000","last_peered":"2024-08-17T17:43:29.952581+0000","last_clean":"2024-08-17T17:43:29.952581+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:43:29.952581+0000","last_undegraded":"2024-08-17T17:43:29.952581+0000","last_fullsized":"2024-08-17T17:43:29.952581+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":37,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.419059+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:43:11.419059+0000","last_peered":"2024-08-17T17:43:11.419059+0000","last_clean":"2024-08-17T17:43:11.419059+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:43:11.419059+0000","last_undegraded":"2024-08-17T17:43:11.419059+0000","last_fullsized":"2024-08-17T17:43:11.419059+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":349,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407500+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:43:11.407500+0000","last_peered":"2024-08-17T17:43:11.407500+0000","last_clean":"2024-08-17T17:43:11.407500+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:43:11.407500+0000","last_undegraded":"2024-08-17T17:43:11.407500+0000","last_fullsized":"2024-08-17T17:43:11.407500+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":312,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:25.696899+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:43:25.696899+0000","last_peered":"2024-08-17T17:43:25.696899+0000","last_clean":"2024-08-17T17:43:25.696899+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:43:25.696899+0000","last_undegraded":"2024-08-17T17:43:25.696899+0000","last_fullsized":"2024-08-17T17:43:25.696899+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":322,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:25.696919+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:43:25.696919+0000","last_peered":"2024-08-17T17:43:25.696919+0000","last_clean":"2024-08-17T17:43:25.696919+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:43:25.696919+0000","last_undegraded":"2024-08-17T17:43:25.696919+0000","last_fullsized":"2024-08-17T17:43:25.696919+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":666,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.949471+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:43:29.949471+0000","last_peered":"2024-08-17T17:43:29.949471+0000","last_clean":"2024-08-17T17:43:29.949471+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:43:29.949471+0000","last_undegraded":"2024-08-17T17:43:29.949471+0000","last_fullsized":"2024-08-17T17:43:29.949471+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":406,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.430334+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:43:11.430334+0000","last_peered":"2024-08-17T17:43:11.430334+0000","last_clean":"2024-08-17T17:43:11.430334+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:43:11.430334+0000","last_undegraded":"2024-08-17T17:43:11.430334+0000","last_fullsized":"2024-08-17T17:43:11.430334+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":190,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:21.225906+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:43:21.225906+0000","last_peered":"2024-08-17T17:43:21.225906+0000","last_clean":"2024-08-17T17:43:21.225906+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:43:21.225906+0000","last_undegraded":"2024-08-17T17:43:21.225906+0000","last_fullsized":"2024-08-17T17:43:21.225906+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":633,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.927695+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:43:29.927695+0000","last_peered":"2024-08-17T17:43:29.927695+0000","last_clean":"2024-08-17T17:43:29.927695+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:43:29.927695+0000","last_undegraded":"2024-08-17T17:43:29.927695+0000","last_fullsized":"2024-08-17T17:43:29.927695+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":691,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.953011+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:43:29.953011+0000","last_peered":"2024-08-17T17:43:29.953011+0000","last_clean":"2024-08-17T17:43:29.953011+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:43:29.953011+0000","last_undegraded":"2024-08-17T17:43:29.953011+0000","last_fullsized":"2024-08-17T17:43:29.953011+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":682,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.930559+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:43:29.930559+0000","last_peered":"2024-08-17T17:43:29.930559+0000","last_clean":"2024-08-17T17:43:29.930559+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:43:29.930559+0000","last_undegraded":"2024-08-17T17:43:29.930559+0000","last_fullsized":"2024-08-17T17:43:29.930559+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413561+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:43:11.413561+0000","last_peered":"2024-08-17T17:43:11.413561+0000","last_clean":"2024-08-17T17:43:11.413561+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:43:11.413561+0000","last_undegraded":"2024-08-17T17:43:11.413561+0000","last_fullsized":"2024-08-17T17:43:11.413561+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":687,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.930866+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:43:29.930866+0000","last_peered":"2024-08-17T17:43:29.930866+0000","last_clean":"2024-08-17T17:43:29.930866+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:43:29.930866+0000","last_undegraded":"2024-08-17T17:43:29.930866+0000","last_fullsized":"2024-08-17T17:43:29.930866+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":665,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952743+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:43:29.952743+0000","last_peered":"2024-08-17T17:43:29.952743+0000","last_clean":"2024-08-17T17:43:29.952743+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:43:29.952743+0000","last_undegraded":"2024-08-17T17:43:29.952743+0000","last_fullsized":"2024-08-17T17:43:29.952743+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":668,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.941787+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:43:29.941787+0000","last_peered":"2024-08-17T17:43:29.941787+0000","last_clean":"2024-08-17T17:43:29.941787+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:43:29.941787+0000","last_undegraded":"2024-08-17T17:43:29.941787+0000","last_fullsized":"2024-08-17T17:43:29.941787+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":683,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.927380+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:43:29.927380+0000","last_peered":"2024-08-17T17:43:29.927380+0000","last_clean":"2024-08-17T17:43:29.927380+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:43:29.927380+0000","last_undegraded":"2024-08-17T17:43:29.927380+0000","last_fullsized":"2024-08-17T17:43:29.927380+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":296,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407684+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:43:11.407684+0000","last_peered":"2024-08-17T17:43:11.407684+0000","last_clean":"2024-08-17T17:43:11.407684+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:43:11.407684+0000","last_undegraded":"2024-08-17T17:43:11.407684+0000","last_fullsized":"2024-08-17T17:43:11.407684+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":196,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952296+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:43:29.952296+0000","last_peered":"2024-08-17T17:43:29.952296+0000","last_clean":"2024-08-17T17:43:29.952296+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:43:29.952296+0000","last_undegraded":"2024-08-17T17:43:29.952296+0000","last_fullsized":"2024-08-17T17:43:29.952296+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.954828+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:43:29.954828+0000","last_peered":"2024-08-17T17:43:29.954828+0000","last_clean":"2024-08-17T17:43:29.954828+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:43:29.954828+0000","last_undegraded":"2024-08-17T17:43:29.954828+0000","last_fullsized":"2024-08-17T17:43:29.954828+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":296,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407845+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:43:11.407845+0000","last_peered":"2024-08-17T17:43:11.407845+0000","last_clean":"2024-08-17T17:43:11.407845+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:43:11.407845+0000","last_undegraded":"2024-08-17T17:43:11.407845+0000","last_fullsized":"2024-08-17T17:43:11.407845+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":193,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.413320+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:43:11.413320+0000","last_peered":"2024-08-17T17:43:11.413320+0000","last_clean":"2024-08-17T17:43:11.413320+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:43:11.413320+0000","last_undegraded":"2024-08-17T17:43:11.413320+0000","last_fullsized":"2024-08-17T17:43:11.413320+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":296,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407645+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:43:11.407645+0000","last_peered":"2024-08-17T17:43:11.407645+0000","last_clean":"2024-08-17T17:43:11.407645+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:43:11.407645+0000","last_undegraded":"2024-08-17T17:43:11.407645+0000","last_fullsized":"2024-08-17T17:43:11.407645+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":203,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.417841+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:43:11.417841+0000","last_peered":"2024-08-17T17:43:11.417841+0000","last_clean":"2024-08-17T17:43:11.417841+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:43:11.417841+0000","last_undegraded":"2024-08-17T17:43:11.417841+0000","last_fullsized":"2024-08-17T17:43:11.417841+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":689,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928122+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:43:29.928122+0000","last_peered":"2024-08-17T17:43:29.928122+0000","last_clean":"2024-08-17T17:43:29.928122+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:43:29.928122+0000","last_undegraded":"2024-08-17T17:43:29.928122+0000","last_fullsized":"2024-08-17T17:43:29.928122+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":690,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.928167+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:43:29.928167+0000","last_peered":"2024-08-17T17:43:29.928167+0000","last_clean":"2024-08-17T17:43:29.928167+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:43:29.928167+0000","last_undegraded":"2024-08-17T17:43:29.928167+0000","last_fullsized":"2024-08-17T17:43:29.928167+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":311,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.407295+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:43:11.407295+0000","last_peered":"2024-08-17T17:43:11.407295+0000","last_clean":"2024-08-17T17:43:11.407295+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:43:11.407295+0000","last_undegraded":"2024-08-17T17:43:11.407295+0000","last_fullsized":"2024-08-17T17:43:11.407295+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":709,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.952920+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:43:29.952920+0000","last_peered":"2024-08-17T17:43:29.952920+0000","last_clean":"2024-08-17T17:43:29.952920+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:43:29.952920+0000","last_undegraded":"2024-08-17T17:43:29.952920+0000","last_fullsized":"2024-08-17T17:43:29.952920+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1694433304,"num_objects":25856,"num_object_clones":0,"num_object_copies":77568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4615,"num_objects_misplaced":1575,"num_objects_unfound":0,"num_objects_dirty":25856,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224130,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1609,"num_bytes_recovered":105447424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5484130304,"data_stored":5484118088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126475,"ondisk_log_size":126475,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201761,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1660060,"kb_used_data":848788,"kb_used_omap":1,"kb_used_meta":811262,"kb_avail":92085092,"statfs":{"total":95995035648,"available":94295134208,"internally_reserved":0,"allocated":869158912,"data_stored":867165264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":830732747},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922257,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":736008,"kb_used_data":337472,"kb_used_omap":7,"kb_used_meta":398520,"kb_avail":93009144,"statfs":{"total":95995035648,"available":95241363456,"internally_reserved":0,"allocated":345571328,"data_stored":343597102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":408085174},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085781,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1409360,"kb_used_data":947336,"kb_used_omap":7,"kb_used_meta":461944,"kb_avail":92335792,"statfs":{"total":95995035648,"available":94551851008,"internally_reserved":0,"allocated":970072064,"data_stored":968086947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":473031097},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986671,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1062484,"kb_used_data":327500,"kb_used_omap":1,"kb_used_meta":734974,"kb_avail":92682668,"statfs":{"total":95995035648,"available":94907052032,"internally_reserved":0,"allocated":335360000,"data_stored":333374458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":752613830},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435562,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799428,"kb_used_data":883388,"kb_used_omap":5,"kb_used_meta":916026,"kb_avail":91945724,"statfs":{"total":95995035648,"available":94152421376,"internally_reserved":0,"allocated":904589312,"data_stored":902615898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":938010697},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599089,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123076,"kb_used_data":311164,"kb_used_omap":7,"kb_used_meta":811896,"kb_avail":92622076,"statfs":{"total":95995035648,"available":94845005824,"internally_reserved":0,"allocated":318631936,"data_stored":316657871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":831381677},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762615,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2198780,"kb_used_data":1413236,"kb_used_omap":7,"kb_used_meta":785528,"kb_avail":91546372,"statfs":{"total":95995035648,"available":93743484928,"internally_reserved":0,"allocated":1447153664,"data_stored":1445186401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":804381089},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926141,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1277256,"kb_used_data":419520,"kb_used_omap":7,"kb_used_meta":857720,"kb_avail":92467896,"statfs":{"total":95995035648,"available":94687125504,"internally_reserved":0,"allocated":429588480,"data_stored":427611213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":878305884},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":420216832,"data_stored":420216832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1432752128,"data_stored":1432748056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":308674560,"data_stored":308674560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":889520128,"data_stored":889520128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":324141056,"data_stored":324141056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":934940672,"data_stored":934936600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":330825728,"data_stored":330825728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":843059200,"data_stored":843055128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:43.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:43:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:43.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:43.116 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:43.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:43.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.279 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:43.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:43.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:43.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:43.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:43.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:43.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:43.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:43 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4721/67110 objects degraded (7.035%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:43.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:43 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4721/67110 objects degraded (7.035%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:43.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:43.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:43.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:43.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:43 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4721/67110 objects degraded (7.035%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:43.925 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:43.955 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:44.056 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:44.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:44.161 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:44.171 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:44.176 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:44.188 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:44.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:44.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:44.317 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:43] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-08-17T17:43:44.352 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:44.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:44.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:44.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:44 smithi064 ceph-mon[104516]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 4721/65607 objects degraded (7.196%); 1362/65607 objects misplaced (2.076%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:44.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:44 smithi064 ceph-mon[109246]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 4721/65607 objects degraded (7.196%); 1362/65607 objects misplaced (2.076%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:44.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:44.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:44.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:44.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:44.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:44.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:44 smithi175 ceph-mon[101434]: pgmap v1287: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 4721/65607 objects degraded (7.196%); 1362/65607 objects misplaced (2.076%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:45.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:45.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:45.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:45.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:45.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:45.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:45.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:45.451 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:45.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:45.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:45.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:45.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:45.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:45.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:45.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:45.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:46.097 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:46.132 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:46.234 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:46.261 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:46.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:46.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:46.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:46.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:46.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:46.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:46.810 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:46 smithi175 ceph-mon[101434]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4721/62304 objects degraded (7.577%); 1362/62304 objects misplaced (2.186%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:46.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:46 smithi064 ceph-mon[104516]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4721/62304 objects degraded (7.577%); 1362/62304 objects misplaced (2.186%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:46 smithi064 ceph-mon[109246]: pgmap v1288: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4721/62304 objects degraded (7.577%); 1362/62304 objects misplaced (2.186%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:43:46.949 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:47.050 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:47.077 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:47.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:47.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:47.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:47.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:47.440 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:47.465 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:47.567 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:47.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:47.699 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:47.732 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:47.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:47.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:47.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:48.014 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:48.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:48.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:48.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:48.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:48.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.421 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:48.457 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:43:48.522 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:48.527 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:48.527 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:48.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:48.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:48.684 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[104516]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:43:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1736318228' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4892/52974 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[109246]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1736318228' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4892/52974 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:48.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:48.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:48 smithi175 ceph-mon[101434]: pgmap v1289: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:43:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1736318228' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:48 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4892/52974 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:48.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:48.955 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:49.057 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:49.091 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:49.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:49.192 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:49.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:49.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:49.363 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:49.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.499 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:49.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:49.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.641 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:49.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:49.742 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:49.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:49.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:49.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[104516]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:43:49.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:49.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:49.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:49 smithi064 ceph-mon[109246]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:43:49.768 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:49.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:43:49.870 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:49.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:49.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:49 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:49.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:49 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:49.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:49 smithi175 ceph-mon[101434]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:43:50.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.033 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:50.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:50.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:50.186 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:50.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:50.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:50.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:50.467 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:50.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:50.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:50.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:50.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:50.735 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:50.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:50 smithi064 ceph-mon[104516]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:50.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:50 smithi064 ceph-mon[109246]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:50.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.873 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:50.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:50.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:50 smithi175 ceph-mon[101434]: pgmap v1291: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4892/52974 objects degraded (9.235%); 1104/52974 objects misplaced (2.084%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:50.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:51.015 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:51.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:51.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:51.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:51.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:51.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:51.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:51.558 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.586 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:51.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:51.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:51.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:51.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:51.861 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:51.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:52.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:52.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:52.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:52.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:52.203 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:43:52.203 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:52.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:43:52.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:52.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:52.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:43:52.277 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:52.283 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:52.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:52.317 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:43:52.317 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:52.350 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:43:52.350 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:52.385 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:52.389 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:43:52.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:52.411 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:52.432 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:43:52.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:52.484 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:43:52.484 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:52.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:52.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:52.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:52.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:52 smithi064 ceph-mon[104516]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 876 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/41919 objects degraded (12.214%); 801/41919 objects misplaced (1.911%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:52.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3797773112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:52.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:52 smithi064 ceph-mon[109246]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 876 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/41919 objects degraded (12.214%); 801/41919 objects misplaced (1.911%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:52.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3797773112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:52.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:52.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:52.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:52.849 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:52.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:52.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:43:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:52 smithi175 ceph-mon[101434]: pgmap v1292: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 876 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/41919 objects degraded (12.214%); 801/41919 objects misplaced (1.911%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3797773112' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:52.951 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:52.979 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:53.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:53.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:53.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:43:53.225 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5120/41919 objects degraded (12.214%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1816,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":406,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915895,"num_in_osds":8,"osd_in_since":1723915756,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":98},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":13973,"data_bytes":918624253,"bytes_used":9416966144,"bytes_avail":758543319040,"bytes_total":767960285184,"degraded_objects":5120,"degraded_total":41919,"degraded_ratio":0.12214031823278228,"misplaced_objects":801,"misplaced_total":41919,"misplaced_ratio":0.019108280254777069,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1061219,"recovering_keys_per_sec":0,"num_objects_recovered":162,"num_bytes_recovered":10616832,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":839},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.qahkwa","status":"up:active","gid":24554}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":30,"modified":"2024-08-17T17:42:27.281728+0000","services":{}},"progress_events":{}} 2024-08-17T17:43:53.249 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:53.351 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:53.382 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:53.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:53.526 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:53.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:53.629 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:53.657 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:53.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/995443275' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5120/41919 objects degraded (12.214%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/995443275' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5120/41919 objects degraded (12.214%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:53.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:53.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/995443275' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5120/41919 objects degraded (12.214%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:53.926 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:53.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:54.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:54.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:54.162 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:54.176 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:43:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:43:53] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-08-17T17:43:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:54.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:54.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:54.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:54.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:54.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:54.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.483 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:54.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:54.584 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:54.620 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:54.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:54.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:54 smithi064 ceph-mon[104516]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 845 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/40425 objects degraded (12.665%); 801/40425 objects misplaced (1.981%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:54.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:54 smithi064 ceph-mon[109246]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 845 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/40425 objects degraded (12.665%); 801/40425 objects misplaced (1.981%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:54.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:54.889 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:54.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:54.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:54 smithi175 ceph-mon[101434]: pgmap v1293: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 845 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 5120/40425 objects degraded (12.665%); 801/40425 objects misplaced (1.981%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:54.948 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:43:54.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:54.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:55.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:55.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.188 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:55.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:55.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:55.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:55.370 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:55.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:55.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:55.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:55.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:55.690 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:55.720 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:55.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:55.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:55.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:56.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:56.104 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:56.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:56.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:43:56.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:56.312 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:56 smithi175 ceph-mon[101434]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 5120/37266 objects degraded (13.739%); 801/37266 objects misplaced (2.149%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:56.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:56.438 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[104516]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 5120/37266 objects degraded (13.739%); 801/37266 objects misplaced (2.149%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:56 smithi064 ceph-mon[109246]: pgmap v1294: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 779 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 5120/37266 objects degraded (13.739%); 801/37266 objects misplaced (2.149%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:43:56.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:56.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:56.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:56.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:56.807 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:56.838 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:56.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:56.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:57.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:57.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:57.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:57.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:57.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:57.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:57.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:57.519 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:57.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:57.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:57.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.790 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:57.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:57.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:57.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:58.026 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.060 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:58.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:58.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:58.198 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:58.300 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:58.336 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:58.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:58.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:58.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:58.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:58.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:58 smithi064 ceph-mon[104516]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:43:58.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:58 smithi064 ceph-mon[109246]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:43:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:58 smithi175 ceph-mon[101434]: pgmap v1295: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:43:58.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:58.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:58.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:58.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:59.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:59.038 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:59.088 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:59.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:59.221 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:59.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:59.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.360 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:59.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:59.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:59.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:43:59 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5333/27840 objects degraded (19.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:59.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:43:59 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5333/27840 objects degraded (19.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:59.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:59.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:59.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:43:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:59.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:43:59 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5333/27840 objects degraded (19.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:43:59.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:59.759 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:43:59.759 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169987 8.10 8:089c48ca:::benchmark_data_smithi064_242541_object47494:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78747205600001, 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2428539119999999, 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169987 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:43:59.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9499999999999998e-07 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903872+0000", 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:43:59.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903877+0000", 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9439999999999996e-06 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903881+0000", 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0589999999999996e-06 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904532+0000", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065159099999999998 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904539+0000", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2670000000000003e-06 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.108495+0000", 2024-08-17T17:43:59.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2039566210000001 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.108618+0000", 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012227700000000001 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.109107+0000", 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048971300000000002 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117836+0000", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087290200000000005 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124586+0000", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067496830000000002 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130499+0000", 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059131540000000003 2024-08-17T17:43:59.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130541+0000", 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2181999999999997e-05 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146530+0000", 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015988176999999999 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146592+0000", 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2388e-05 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146724+0000", 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131812 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169991 8.10 8:0d6229e0:::benchmark_data_smithi064_242541_object47498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:43:59.768 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78744501199998, 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2432334940000001, 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169991 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9999999999999998e-07 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903899+0000", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9319999999999999e-06 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903905+0000", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8629999999999996e-06 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903910+0000", 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7380000000000002e-06 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904561+0000", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065160999999999995 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904568+0000", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5899999999999996e-06 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.114585+0000", 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2100170699999999 2024-08-17T17:43:59.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.114710+0000", 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012532 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.115118+0000", 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040799299999999999 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117895+0000", 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027765450000000001 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124599+0000", 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067038530000000001 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130588+0000", 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059897450000000003 2024-08-17T17:43:59.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130681+0000", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2915000000000001e-05 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146948+0000", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016266138999999999 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147007+0000", 2024-08-17T17:43:59.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9379999999999997e-05 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147130+0000", 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012340100000000001 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169992 8.10 8:0e67e8fd:::benchmark_data_smithi064_242541_object47499:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78742743999999, 2024-08-17T17:43:59.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243508251, 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169992 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903915+0000", 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2300000000000002e-07 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903916+0000", 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.345e-06 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903937+0000", 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1189999999999999e-05 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903942+0000", 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6800000000000001e-06 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904593+0000", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065059899999999997 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904599+0000", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5289999999999997e-06 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117967+0000", 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.213367941 2024-08-17T17:43:59.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.118102+0000", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135067 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.118638+0000", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053601699999999996 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124526+0000", 2024-08-17T17:43:59.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058880119999999998 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124606+0000", 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0197000000000005e-05 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130771+0000", 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061646030000000003 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146771+0000", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016000316000000001 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147224+0000", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000452686 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147281+0000", 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7136999999999999e-05 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147423+0000", 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014160900000000001 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170001 8.10 8:0f85a16f:::benchmark_data_smithi064_242541_object47508:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78739479400002, 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244356507, 2024-08-17T17:43:59.784 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170001 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2899999999999999e-07 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903949+0000", 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.635e-06 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903954+0000", 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4399999999999996e-06 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903959+0000", 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1250000000000003e-06 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904625+0000", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066614300000000001 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904631+0000", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.02e-06 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.121580+0000", 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.216949662 2024-08-17T17:43:59.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.121845+0000", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026459999999999998 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.122273+0000", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042810299999999999 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124574+0000", 2024-08-17T17:43:59.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002301179 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130816+0000", 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062412489999999999 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147177+0000", 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016361550999999998 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147515+0000", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000338174 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148054+0000", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053849400000000004 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148108+0000", 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4320000000000002e-05 2024-08-17T17:43:59.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148304+0000", 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019548300000000001 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170081 8.10 8:0e59af9f:::benchmark_data_smithi064_242541_object47588:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:43:59.792 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78737881500001, 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2447100250000001, 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170081 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:43:59.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2099999999999998e-07 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903965+0000", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4219999999999999e-06 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903971+0000", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6619999999999999e-06 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903976+0000", 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.155e-06 2024-08-17T17:43:59.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904655+0000", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067936500000000003 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904662+0000", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.294e-06 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124647+0000", 2024-08-17T17:43:59.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.219984261 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124737+0000", 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0848999999999997e-05 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.125056+0000", 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031911 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130634+0000", 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055775920000000001 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146485+0000", 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015850578000000001 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147467+0000", 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098212400000000002 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147602+0000", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135482 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148496+0000", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00089352699999999997 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148549+0000", 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2936000000000003e-05 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148673+0000", 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012434699999999999 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170094 8.10 8:08c04a44:::benchmark_data_smithi064_242541_object47601:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78736178499997, 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244854007, 2024-08-17T17:43:59.800 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170094 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9200000000000002e-07 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903982+0000", 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8029999999999999e-06 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903987+0000", 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1660000000000002e-06 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903991+0000", 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1699999999999999e-06 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904688+0000", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069637300000000002 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904695+0000", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7429999999999996e-06 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.126968+0000", 2024-08-17T17:43:59.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2222730880000001 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.127086+0000", 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118687 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.127420+0000", 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033324199999999998 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130726+0000", 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033066710000000002 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146901+0000", 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016175168 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147562+0000", 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066013900000000002 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147642+0000", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0129999999999993e-05 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148680+0000", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001038073 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148724+0000", 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4008999999999998e-05 2024-08-17T17:43:59.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148834+0000", 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110253 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170129 8.10 8:08f27c42:::benchmark_data_smithi064_242541_object47636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78734594000002, 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2624799170000001, 2024-08-17T17:43:59.808 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170129 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9799999999999999e-07 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903998+0000", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6300000000000001e-06 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904004+0000", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4670000000000001e-06 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904009+0000", 2024-08-17T17:43:59.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3660000000000002e-06 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904719+0000", 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000710591 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904726+0000", 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5130000000000002e-06 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143356+0000", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.238629918 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143482+0000", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012597899999999999 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143836+0000", 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035482299999999999 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146841+0000", 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003004456 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148040+0000", 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011986900000000001 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148840+0000", 2024-08-17T17:43:59.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080081300000000005 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166184+0000", 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017343715999999999 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166291+0000", 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106687 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166351+0000", 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0102000000000002e-05 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166476+0000", 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012486800000000001 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170285 8.10 8:0860b59f:::benchmark_data_smithi064_242541_object47792:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78732885599999, 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2635653019999999, 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170285 2024-08-17T17:43:59.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2500000000000001e-07 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904019+0000", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3920000000000002e-06 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904026+0000", 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9260000000000003e-06 2024-08-17T17:43:59.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904030+0000", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3340000000000003e-06 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904752+0000", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072161000000000002 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904758+0000", 2024-08-17T17:43:59.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6869999999999997e-06 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.151809+0000", 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.247050346 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.151945+0000", 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013669299999999999 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.152345+0000", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000399527 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163133+0000", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010787711 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.165979+0000", 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028468270000000001 2024-08-17T17:43:59.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166243+0000", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026321800000000001 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166559+0000", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031634700000000003 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166641+0000", 2024-08-17T17:43:59.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2047999999999998e-05 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167435+0000", 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079408199999999999 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167578+0000", 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014322900000000001 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170286 8.10 8:0ea5b216:::benchmark_data_smithi064_242541_object47793:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78730713499999, 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2641885799999999, 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170286 2024-08-17T17:43:59.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1300000000000001e-07 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904037+0000", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.689e-06 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904043+0000", 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4529999999999999e-06 2024-08-17T17:43:59.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904048+0000", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7090000000000001e-06 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904783+0000", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073487800000000001 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904789+0000", 2024-08-17T17:43:59.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2400000000000004e-06 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.154791+0000", 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.250002142 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.154906+0000", 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114443 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.155276+0000", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037086200000000001 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.165936+0000", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010659221999999999 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166518+0000", 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058253700000000005 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166600+0000", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1546999999999997e-05 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167623+0000", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010228850000000001 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167861+0000", 2024-08-17T17:43:59.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000238784 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167925+0000", 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4028000000000004e-05 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.168223+0000", 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029784800000000003 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170365 8.10 8:08eecb7e:::benchmark_data_smithi064_242541_object47872:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78728945, 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.276395232, 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.838 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170365 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904053+0000", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7399999999999999e-07 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904054+0000", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.511e-06 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904059+0000", 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1429999999999997e-06 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904069+0000", 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6190000000000006e-06 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904812+0000", 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00074339899999999995 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904819+0000", 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1909999999999997e-06 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163165+0000", 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2583468250000001 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163287+0000", 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012170100000000001 2024-08-17T17:43:59.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163679+0000", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039162499999999999 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.168831+0000", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051521199999999996 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172125+0000", 2024-08-17T17:43:59.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032937069999999999 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172269+0000", 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014436700000000001 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.176836+0000", 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045666930000000001 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180269+0000", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034337500000000002 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180323+0000", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3349e-05 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180448+0000", 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495799999999999 2024-08-17T17:43:59.845 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170367 8.10 8:0f498d28:::benchmark_data_smithi064_242541_object47874:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 386.78726834499997, 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2886117530000001, 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.846 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170367 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:43:59.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.03e-07 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904075+0000", 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.398e-06 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904085+0000", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4590000000000004e-06 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904089+0000", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904905+0000", 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081554800000000003 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904912+0000", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1600000000000001e-06 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172352+0000", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.267439948 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172559+0000", 2024-08-17T17:43:59.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020787099999999999 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.173288+0000", 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072807099999999995 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.181788+0000", 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085003179999999998 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.182960+0000", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011723339999999999 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.183025+0000", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4449999999999994e-05 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.184212+0000", 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011870310000000001 2024-08-17T17:43:59.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192457+0000", 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082452319999999999 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192537+0000", 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0438000000000005e-05 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192685+0000", 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000147892 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143320 9.6 e397/375 9:66dc7620:::benchmark_data_smithi064_275711_object18325:head v 397'8971, mlcod=397'8968)", 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.834296549, 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36413622600000001, 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.854 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857046+0000", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8099999999999999e-07 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857049+0000", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.579e-06 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857055+0000", 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.122e-06 2024-08-17T17:43:59.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857060+0000", 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3170000000000003e-06 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.859788+0000", 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027285529999999999 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.859804+0000", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6273000000000002e-05 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221126+0000", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36132131200000001 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221182+0000", 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5788999999999997e-05 2024-08-17T17:43:59.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143328 9.6 e397/375 9:68bcc8f8:::benchmark_data_smithi064_275711_object18333:head v 397'8972, mlcod=397'8968)", 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.828024958, 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35789923299999998, 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863318+0000", 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.564e-06 2024-08-17T17:43:59.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863326+0000", 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9959999999999993e-06 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863341+0000", 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4409999999999999e-05 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863349+0000", 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0430000000000008e-06 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.872140+0000", 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087913839999999993 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.872219+0000", 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8678999999999997e-05 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221190+0000", 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34897107399999999 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221216+0000", 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6083000000000001e-05 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143330 9.6 e397/375 9:6a00a6c4:::benchmark_data_smithi064_275711_object18335:head v 397'8973, mlcod=397'8968)", 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.822984443, 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35288467099999998, 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868358+0000", 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0100000000000005e-07 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868364+0000", 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7679999999999997e-06 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868374+0000", 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0087e-05 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868381+0000", 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7689999999999999e-06 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881427+0000", 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013046713 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881449+0000", 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1970000000000001e-05 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221219+0000", 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33976983399999999 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221242+0000", 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2929e-05 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143338 9.0 9:0157aaa0:::benchmark_data_smithi064_275711_object18343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:43:59.869 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.815092893, 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34528550899999999, 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143338 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876251+0000", 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9989999999999998e-06 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876256+0000", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876270+0000", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4054e-05 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876278+0000", 2024-08-17T17:43:59.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3469999999999999e-06 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876879+0000", 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060110999999999997 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876989+0000", 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110305 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877163+0000", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017362599999999999 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884715+0000", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075518879999999997 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884860+0000", 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014478099999999999 2024-08-17T17:43:59.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221121+0000", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.336261635 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221235+0000", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011311600000000001 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221534+0000", 2024-08-17T17:43:59.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029992899999999999 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143341 9.0 9:0b34feb9:::benchmark_data_smithi064_275711_object18346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.814442118, 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34476353100000001, 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.876 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143341 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876901+0000", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.035e-06 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876906+0000", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9690000000000002e-06 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876917+0000", 2024-08-17T17:43:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1211e-05 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876924+0000", 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7000000000000002e-06 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880772+0000", 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038480379999999998 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880868+0000", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6372000000000005e-05 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881067+0000", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019910500000000001 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884867+0000", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003799714 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.885267+0000", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039973899999999998 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221584+0000", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.336317596 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221630+0000", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5741999999999997e-05 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221663+0000", 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3309999999999998e-05 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143337 9.f e397/375 9:ffec66a7:::benchmark_data_smithi064_275711_object18342:head v 397'8890, mlcod=397'8886)", 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.877071+0000", 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.814270771, 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34415817999999998, 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.882 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877071+0000", 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877071+0000", 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877072+0000", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2900000000000003e-07 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877079+0000", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1330000000000001e-06 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877088+0000", 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7190000000000007e-06 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877093+0000", 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8220000000000003e-06 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.885615+0000", 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085213159999999993 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.885632+0000", 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7056000000000001e-05 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890135+0000", 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045028170000000001 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221229+0000", 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33109458800000002 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.886 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143343 9.6 e397/375 9:676e64a4:::benchmark_data_smithi064_275711_object18348:head v 397'8974, mlcod=397'8969)", 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.880491+0000", 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.81085083799999, 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34079488600000002, 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880491+0000", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880491+0000", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880492+0000", 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.102e-06 2024-08-17T17:43:59.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880500+0000", 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4490000000000002e-06 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880513+0000", 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3275000000000001e-05 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880520+0000", 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1420000000000004e-06 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.888085+0000", 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075649699999999999 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.888150+0000", 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5454e-05 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221251+0000", 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33310020699999998 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221286+0000", 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5287000000000003e-05 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.891 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143347 9.0 9:002cf2b1:::benchmark_data_smithi064_275711_object18352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.80082841700001, 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.349745097, 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143347 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890515+0000", 2024-08-17T17:43:59.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3790000000000001e-06 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890519+0000", 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4490000000000001e-06 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.225664+0000", 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.335145111 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.225681+0000", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7027000000000001e-05 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226202+0000", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052065800000000001 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226304+0000", 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101618 2024-08-17T17:43:59.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226412+0000", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010857900000000001 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.238847+0000", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012434822999999999 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.238864+0000", 2024-08-17T17:43:59.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6684000000000002e-05 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240165+0000", 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013013639999999999 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240218+0000", 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2899999999999998e-05 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240259+0000", 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0504999999999999e-05 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143352 9.a e397/375 9:541fecb1:::benchmark_data_smithi064_275711_object18357:head v 397'8930, mlcod=397'8928)", 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.897050+0000", 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.794292153, 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33881088500000001, 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:43:59.898 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.897050+0000", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.897050+0000", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:43:59.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.897051+0000", 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1310000000000001e-06 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.897059+0000", 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0490000000000004e-06 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221209+0000", 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32415012999999998 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221221+0000", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2429000000000001e-05 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221289+0000", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7127999999999998e-05 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221310+0000", 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1582000000000001e-05 2024-08-17T17:43:59.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.235719+0000", 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014409279000000001 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.235861+0000", 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014115699999999999 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.902 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.903 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:43:59.903 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:43:59.903 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:43:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.904 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:43:59.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:00.005 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:00.040 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:00.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:00.178 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:00.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:00.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:00.428 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:00.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:00.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:00.599 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:00.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:00.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:00 smithi175 ceph-mon[101434]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 906 KiB/s, 14 objects/s recovering 2024-08-17T17:44:00.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:00 smithi064 ceph-mon[104516]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 906 KiB/s, 14 objects/s recovering 2024-08-17T17:44:00.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:00 smithi064 ceph-mon[109246]: pgmap v1296: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 583 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5333/27840 objects degraded (19.156%); 509/27840 objects misplaced (1.828%); 906 KiB/s, 14 objects/s recovering 2024-08-17T17:44:00.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:00.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:00.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:00.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:00.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:01.012 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:01.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:01.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:01.145 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:01.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:01.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:01.408 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:01.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:01.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:01.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:01.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:01.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:01.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:01.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:44:01.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:01.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:01.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.037 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:02.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:02.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:02.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:02.281 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:02.317 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:02.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:02.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:02.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:02.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:02.592 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:02 smithi064 ceph-mon[104516]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 345 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5497/16425 objects degraded (33.467%); 250/16425 objects misplaced (1.522%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:44:02.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:02 smithi064 ceph-mon[109246]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 345 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5497/16425 objects degraded (33.467%); 250/16425 objects misplaced (1.522%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:44:02.695 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:02.718 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:02.820 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:02.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:02.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:02 smithi175 ceph-mon[101434]: pgmap v1297: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 345 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5497/16425 objects degraded (33.467%); 250/16425 objects misplaced (1.522%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:44:02.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:02.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:03.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:03.129 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:03.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:03.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:03.256 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:03.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.408 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:03.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:03.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:03.532 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:03.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:03.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:03.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:03.899 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:03.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:04.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:04.064 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:04.165 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:04.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:04.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:04.297 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:04.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:04.379 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:03] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-08-17T17:44:04.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.444 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:44:04.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:04.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[104516]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 312 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 5497/14823 objects degraded (37.084%); 250/14823 objects misplaced (1.687%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:44:04.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5497/14823 objects degraded (37.084%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:04.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[109246]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 312 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 5497/14823 objects degraded (37.084%); 250/14823 objects misplaced (1.687%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:44:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5497/14823 objects degraded (37.084%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:04.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:04 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:04.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.752 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:04.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:04 smithi175 ceph-mon[101434]: pgmap v1298: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 312 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 5497/14823 objects degraded (37.084%); 250/14823 objects misplaced (1.687%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:44:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:04 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5497/14823 objects degraded (37.084%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:04 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:04.999 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:05.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:05.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.176 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:05.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:05.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:05.318 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:05.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:05.449 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:05.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:05.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:05.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[104516]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:05.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:05 smithi064 ceph-mon[109246]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:44:05.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:05.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:05.856 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:05.867 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:05.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:05 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:05.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2917599923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:05.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:05 smithi175 ceph-mon[101434]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:44:05.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:05.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:06.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:06.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.140 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:06.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.419 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:06.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:06.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:06.548 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:06.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:06.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:06.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:06 smithi064 ceph-mon[104516]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 250 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5497/11856 objects degraded (46.365%); 250/11856 objects misplaced (2.109%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:44:06.756 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:06 smithi064 ceph-mon[109246]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 250 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5497/11856 objects degraded (46.365%); 250/11856 objects misplaced (2.109%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:44:06.763 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:06.788 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 124992 objects 2024-08-17T17:44:06.788 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :152.752 2024-08-17T17:44:06.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:06 smithi175 ceph-mon[101434]: pgmap v1300: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 250 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5497/11856 objects degraded (46.365%); 250/11856 objects misplaced (2.109%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:44:06.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.977 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:44:06.978 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:44:07.004 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-08-17T17:44:07.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-08-17T17:44:07.055 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:07.076 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:07.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:07.199 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:07.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:07.425 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:07.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:07.547 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:07.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:07.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:07.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:07.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:07.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:07.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:44:07.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:07.969 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:07.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:08.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:08.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:08.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:08.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:08.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:08.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:08.340 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:08.390 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1300,"stamp":"2024-08-17T17:44:05.334319+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":261887997,"num_objects":3952,"num_object_clones":0,"num_object_copies":11856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5497,"num_objects_misplaced":250,"num_objects_unfound":0,"num_objects_dirty":3952,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":246970,"num_write_kb":8012114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2173,"num_bytes_recovered":142874851,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129079,"ondisk_log_size":129079,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7201652,"kb_used_data":1347060,"kb_used_omap":46,"kb_used_meta":5854289,"kb_avail":742759564,"statfs":{"total":767960285184,"available":760585793536,"internally_reserved":0,"allocated":1379389440,"data_stored":1363527602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":5994792179},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,4,8,8,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-555089920,"num_objects":-8470,"num_object_clones":0,"num_object_copies":-25410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":377,"num_objects_misplaced":-551,"num_objects_unfound":0,"num_objects_dirty":-8470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8470,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":11403264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004536"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":675,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461932+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:44:04.461932+0000","last_peered":"2024-08-17T17:44:04.461932+0000","last_clean":"2024-08-17T17:44:04.461932+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:44:04.461932+0000","last_undegraded":"2024-08-17T17:44:04.461932+0000","last_fullsized":"2024-08-17T17:44:04.461932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":407,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935527+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:43:29.935527+0000","last_peered":"2024-08-17T17:43:29.935527+0000","last_clean":"2024-08-17T17:43:29.935527+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:43:29.935527+0000","last_undegraded":"2024-08-17T17:43:29.935527+0000","last_fullsized":"2024-08-17T17:43:29.935527+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.933648+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:43:29.933648+0000","last_peered":"2024-08-17T17:43:29.933648+0000","last_clean":"2024-08-17T17:43:29.933648+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:43:29.933648+0000","last_undegraded":"2024-08-17T17:43:29.933648+0000","last_fullsized":"2024-08-17T17:43:29.933648+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":665,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.471755+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:43:48.471755+0000","last_peered":"2024-08-17T17:43:48.471755+0000","last_clean":"2024-08-17T17:43:48.471755+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:43:48.471755+0000","last_undegraded":"2024-08-17T17:43:48.471755+0000","last_fullsized":"2024-08-17T17:43:48.471755+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":665,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.477294+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:43:48.477294+0000","last_peered":"2024-08-17T17:43:48.477294+0000","last_clean":"2024-08-17T17:43:48.477294+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:43:48.477294+0000","last_undegraded":"2024-08-17T17:43:48.477294+0000","last_fullsized":"2024-08-17T17:43:48.477294+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":393,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935790+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:43:29.935790+0000","last_peered":"2024-08-17T17:43:29.935790+0000","last_clean":"2024-08-17T17:43:29.935790+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:43:29.935790+0000","last_undegraded":"2024-08-17T17:43:29.935790+0000","last_fullsized":"2024-08-17T17:43:29.935790+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":675,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461189+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:44:04.461189+0000","last_peered":"2024-08-17T17:44:04.461189+0000","last_clean":"2024-08-17T17:44:04.461189+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:44:04.461189+0000","last_undegraded":"2024-08-17T17:44:04.461189+0000","last_fullsized":"2024-08-17T17:44:04.461189+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":342,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.465261+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:43:48.465261+0000","last_peered":"2024-08-17T17:43:48.465261+0000","last_clean":"2024-08-17T17:43:48.465261+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:43:48.465261+0000","last_undegraded":"2024-08-17T17:43:48.465261+0000","last_fullsized":"2024-08-17T17:43:48.465261+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"406'15371","reported_seq":19551,"reported_epoch":406,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:01.222538+0000","last_change":"2024-08-17T17:41:35.239044+0000","last_active":"2024-08-17T17:44:01.222538+0000","last_peered":"2024-08-17T17:44:01.222538+0000","last_clean":"2024-08-17T17:41:30.558478+0000","last_became_active":"2024-08-17T17:41:32.793201+0000","last_became_peered":"2024-08-17T17:41:32.793201+0000","last_unstale":"2024-08-17T17:44:01.222538+0000","last_undegraded":"2024-08-17T17:41:35.232757+0000","last_fullsized":"2024-08-17T17:41:32.604712+0000","mapping_epoch":393,"log_start":"390'6200","ondisk_log_start":"390'6200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9171,"log_dups_size":0,"ondisk_log_size":9171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5497,"num_objects_misplaced":250,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15371,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":250},{"shards":"0,5","objects":-5474}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935928+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:43:29.935928+0000","last_peered":"2024-08-17T17:43:29.935928+0000","last_clean":"2024-08-17T17:43:29.935928+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:43:29.935928+0000","last_undegraded":"2024-08-17T17:43:29.935928+0000","last_fullsized":"2024-08-17T17:43:29.935928+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":201,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.932926+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:43:29.932926+0000","last_peered":"2024-08-17T17:43:29.932926+0000","last_clean":"2024-08-17T17:43:29.932926+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:43:29.932926+0000","last_undegraded":"2024-08-17T17:43:29.932926+0000","last_fullsized":"2024-08-17T17:43:29.932926+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"406'15345","reported_seq":15419,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.530356+0000","last_change":"2024-08-17T17:39:46.548255+0000","last_active":"2024-08-17T17:44:01.530356+0000","last_peered":"2024-08-17T17:44:01.530356+0000","last_clean":"2024-08-17T17:44:01.530356+0000","last_became_active":"2024-08-17T17:39:46.547975+0000","last_became_peered":"2024-08-17T17:39:46.547975+0000","last_unstale":"2024-08-17T17:44:01.530356+0000","last_undegraded":"2024-08-17T17:44:01.530356+0000","last_fullsized":"2024-08-17T17:44:01.530356+0000","mapping_epoch":375,"log_start":"396'8200","ondisk_log_start":"396'8200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7145,"log_dups_size":0,"ondisk_log_size":7145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:20.468448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15345,"num_write_kb":500480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":404,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.457716+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:43:48.457716+0000","last_peered":"2024-08-17T17:43:48.457716+0000","last_clean":"2024-08-17T17:43:48.457716+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:43:48.457716+0000","last_undegraded":"2024-08-17T17:43:48.457716+0000","last_fullsized":"2024-08-17T17:43:48.457716+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":298,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936144+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:43:29.936144+0000","last_peered":"2024-08-17T17:43:29.936144+0000","last_clean":"2024-08-17T17:43:29.936144+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:43:29.936144+0000","last_undegraded":"2024-08-17T17:43:29.936144+0000","last_fullsized":"2024-08-17T17:43:29.936144+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"406'15352","reported_seq":15391,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.251864+0000","last_change":"2024-08-17T17:39:46.537111+0000","last_active":"2024-08-17T17:44:01.251864+0000","last_peered":"2024-08-17T17:44:01.251864+0000","last_clean":"2024-08-17T17:44:01.251864+0000","last_became_active":"2024-08-17T17:39:46.536747+0000","last_became_peered":"2024-08-17T17:39:46.536747+0000","last_unstale":"2024-08-17T17:44:01.251864+0000","last_undegraded":"2024-08-17T17:44:01.251864+0000","last_fullsized":"2024-08-17T17:44:01.251864+0000","mapping_epoch":375,"log_start":"390'6500","ondisk_log_start":"390'6500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:18.381107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18612224,"num_objects":284,"num_object_clones":0,"num_object_copies":852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15352,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":421,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.466816+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:43:48.466816+0000","last_peered":"2024-08-17T17:43:48.466816+0000","last_clean":"2024-08-17T17:43:48.466816+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:43:48.466816+0000","last_undegraded":"2024-08-17T17:43:48.466816+0000","last_fullsized":"2024-08-17T17:43:48.466816+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":322,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.460360+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:43:48.460360+0000","last_peered":"2024-08-17T17:43:48.460360+0000","last_clean":"2024-08-17T17:43:48.460360+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:43:48.460360+0000","last_undegraded":"2024-08-17T17:43:48.460360+0000","last_fullsized":"2024-08-17T17:43:48.460360+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"406'15395","reported_seq":15436,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.257202+0000","last_change":"2024-08-17T17:39:46.770631+0000","last_active":"2024-08-17T17:44:01.257202+0000","last_peered":"2024-08-17T17:44:01.257202+0000","last_clean":"2024-08-17T17:44:01.257202+0000","last_became_active":"2024-08-17T17:39:46.769878+0000","last_became_peered":"2024-08-17T17:39:46.769878+0000","last_unstale":"2024-08-17T17:44:01.257202+0000","last_undegraded":"2024-08-17T17:44:01.257202+0000","last_fullsized":"2024-08-17T17:44:01.257202+0000","mapping_epoch":375,"log_start":"390'6500","ondisk_log_start":"390'6500","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:13.213173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18022400,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15395,"num_write_kb":501440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":201,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.933310+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:43:29.933310+0000","last_peered":"2024-08-17T17:43:29.933310+0000","last_clean":"2024-08-17T17:43:29.933310+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:43:29.933310+0000","last_undegraded":"2024-08-17T17:43:29.933310+0000","last_fullsized":"2024-08-17T17:43:29.933310+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":679,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.476578+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:43:48.476578+0000","last_peered":"2024-08-17T17:43:48.476578+0000","last_clean":"2024-08-17T17:43:48.476578+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:43:48.476578+0000","last_undegraded":"2024-08-17T17:43:48.476578+0000","last_fullsized":"2024-08-17T17:43:48.476578+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"407'15636","reported_seq":15713,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.481389+0000","last_change":"2024-08-17T17:39:46.550963+0000","last_active":"2024-08-17T17:44:04.481389+0000","last_peered":"2024-08-17T17:44:04.481389+0000","last_clean":"2024-08-17T17:44:04.481389+0000","last_became_active":"2024-08-17T17:39:46.550632+0000","last_became_peered":"2024-08-17T17:39:46.550632+0000","last_unstale":"2024-08-17T17:44:04.481389+0000","last_undegraded":"2024-08-17T17:44:04.481389+0000","last_fullsized":"2024-08-17T17:44:04.481389+0000","mapping_epoch":375,"log_start":"397'8600","ondisk_log_start":"397'8600","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:06.049315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15636,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":762,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.458028+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:43:48.458028+0000","last_peered":"2024-08-17T17:43:48.458028+0000","last_clean":"2024-08-17T17:43:48.458028+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:43:48.458028+0000","last_undegraded":"2024-08-17T17:43:48.458028+0000","last_fullsized":"2024-08-17T17:43:48.458028+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":697,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465260+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:44:04.465260+0000","last_peered":"2024-08-17T17:44:04.465260+0000","last_clean":"2024-08-17T17:44:04.465260+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:44:04.465260+0000","last_undegraded":"2024-08-17T17:44:04.465260+0000","last_fullsized":"2024-08-17T17:44:04.465260+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":322,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936624+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:43:29.936624+0000","last_peered":"2024-08-17T17:43:29.936624+0000","last_clean":"2024-08-17T17:43:29.936624+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:43:29.936624+0000","last_undegraded":"2024-08-17T17:43:29.936624+0000","last_fullsized":"2024-08-17T17:43:29.936624+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"406'14962","reported_seq":15037,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.535487+0000","last_change":"2024-08-17T17:39:46.770125+0000","last_active":"2024-08-17T17:44:01.535487+0000","last_peered":"2024-08-17T17:44:01.535487+0000","last_clean":"2024-08-17T17:44:01.535487+0000","last_became_active":"2024-08-17T17:39:46.769677+0000","last_became_peered":"2024-08-17T17:39:46.769677+0000","last_unstale":"2024-08-17T17:44:01.535487+0000","last_undegraded":"2024-08-17T17:44:01.535487+0000","last_fullsized":"2024-08-17T17:44:01.535487+0000","mapping_epoch":375,"log_start":"396'7800","ondisk_log_start":"396'7800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7162,"log_dups_size":0,"ondisk_log_size":7162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.543931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14962,"num_write_kb":487616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":423,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451257+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:43:48.451257+0000","last_peered":"2024-08-17T17:43:48.451257+0000","last_clean":"2024-08-17T17:43:48.451257+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:43:48.451257+0000","last_undegraded":"2024-08-17T17:43:48.451257+0000","last_fullsized":"2024-08-17T17:43:48.451257+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":328,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936216+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:43:29.936216+0000","last_peered":"2024-08-17T17:43:29.936216+0000","last_clean":"2024-08-17T17:43:29.936216+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:43:29.936216+0000","last_undegraded":"2024-08-17T17:43:29.936216+0000","last_fullsized":"2024-08-17T17:43:29.936216+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":298,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936603+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:43:29.936603+0000","last_peered":"2024-08-17T17:43:29.936603+0000","last_clean":"2024-08-17T17:43:29.936603+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:43:29.936603+0000","last_undegraded":"2024-08-17T17:43:29.936603+0000","last_fullsized":"2024-08-17T17:43:29.936603+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"406'15133","reported_seq":15175,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:00.895675+0000","last_change":"2024-08-17T17:39:46.539403+0000","last_active":"2024-08-17T17:44:00.895675+0000","last_peered":"2024-08-17T17:44:00.895675+0000","last_clean":"2024-08-17T17:44:00.895675+0000","last_became_active":"2024-08-17T17:39:46.539015+0000","last_became_peered":"2024-08-17T17:39:46.539015+0000","last_unstale":"2024-08-17T17:44:00.895675+0000","last_undegraded":"2024-08-17T17:44:00.895675+0000","last_fullsized":"2024-08-17T17:44:00.895675+0000","mapping_epoch":375,"log_start":"396'8138","ondisk_log_start":"396'8138","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:08.377895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15133,"num_write_kb":494208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":298,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936068+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:43:29.936068+0000","last_peered":"2024-08-17T17:43:29.936068+0000","last_clean":"2024-08-17T17:43:29.936068+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:43:29.936068+0000","last_undegraded":"2024-08-17T17:43:29.936068+0000","last_fullsized":"2024-08-17T17:43:29.936068+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":313,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936961+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:43:29.936961+0000","last_peered":"2024-08-17T17:43:29.936961+0000","last_clean":"2024-08-17T17:43:29.936961+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:43:29.936961+0000","last_undegraded":"2024-08-17T17:43:29.936961+0000","last_fullsized":"2024-08-17T17:43:29.936961+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"407'15258","reported_seq":15335,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.463946+0000","last_change":"2024-08-17T17:39:46.770140+0000","last_active":"2024-08-17T17:44:04.463946+0000","last_peered":"2024-08-17T17:44:04.463946+0000","last_clean":"2024-08-17T17:44:04.463946+0000","last_became_active":"2024-08-17T17:39:46.769773+0000","last_became_peered":"2024-08-17T17:39:46.769773+0000","last_unstale":"2024-08-17T17:44:04.463946+0000","last_undegraded":"2024-08-17T17:44:04.463946+0000","last_fullsized":"2024-08-17T17:44:04.463946+0000","mapping_epoch":375,"log_start":"396'8200","ondisk_log_start":"396'8200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:18.777907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7733248,"num_objects":118,"num_object_clones":0,"num_object_copies":354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15258,"num_write_kb":492032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":423,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.452152+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:43:48.452152+0000","last_peered":"2024-08-17T17:43:48.452152+0000","last_clean":"2024-08-17T17:43:48.452152+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:43:48.452152+0000","last_undegraded":"2024-08-17T17:43:48.452152+0000","last_fullsized":"2024-08-17T17:43:48.452152+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":677,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460769+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:44:04.460769+0000","last_peered":"2024-08-17T17:44:04.460769+0000","last_clean":"2024-08-17T17:44:04.460769+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:44:04.460769+0000","last_undegraded":"2024-08-17T17:44:04.460769+0000","last_fullsized":"2024-08-17T17:44:04.460769+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":663,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.462560+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:43:48.462560+0000","last_peered":"2024-08-17T17:43:48.462560+0000","last_clean":"2024-08-17T17:43:48.462560+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:43:48.462560+0000","last_undegraded":"2024-08-17T17:43:48.462560+0000","last_fullsized":"2024-08-17T17:43:48.462560+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"407'15677","reported_seq":15753,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.484706+0000","last_change":"2024-08-17T17:39:46.536779+0000","last_active":"2024-08-17T17:44:04.484706+0000","last_peered":"2024-08-17T17:44:04.484706+0000","last_clean":"2024-08-17T17:44:04.484706+0000","last_became_active":"2024-08-17T17:39:46.536473+0000","last_became_peered":"2024-08-17T17:39:46.536473+0000","last_unstale":"2024-08-17T17:44:04.484706+0000","last_undegraded":"2024-08-17T17:44:04.484706+0000","last_fullsized":"2024-08-17T17:44:04.484706+0000","mapping_epoch":375,"log_start":"397'8700","ondisk_log_start":"397'8700","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:44:30.170299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8060928,"num_objects":123,"num_object_clones":0,"num_object_copies":369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15677,"num_write_kb":505600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":745,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935342+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:43:29.935342+0000","last_peered":"2024-08-17T17:43:29.935342+0000","last_clean":"2024-08-17T17:43:29.935342+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:43:29.935342+0000","last_undegraded":"2024-08-17T17:43:29.935342+0000","last_fullsized":"2024-08-17T17:43:29.935342+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1000,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.275551+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:43:46.275551+0000","last_peered":"2024-08-17T17:43:46.275551+0000","last_clean":"2024-08-17T17:43:46.275551+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:43:46.275551+0000","last_undegraded":"2024-08-17T17:43:46.275551+0000","last_fullsized":"2024-08-17T17:43:46.275551+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":661,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.464001+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:44:04.464001+0000","last_peered":"2024-08-17T17:44:04.464001+0000","last_clean":"2024-08-17T17:44:04.464001+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:44:04.464001+0000","last_undegraded":"2024-08-17T17:44:04.464001+0000","last_fullsized":"2024-08-17T17:44:04.464001+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936539+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:43:29.936539+0000","last_peered":"2024-08-17T17:43:29.936539+0000","last_clean":"2024-08-17T17:43:29.936539+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:43:29.936539+0000","last_undegraded":"2024-08-17T17:43:29.936539+0000","last_fullsized":"2024-08-17T17:43:29.936539+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"406'15385","reported_seq":15459,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.149288+0000","last_change":"2024-08-17T17:39:46.551179+0000","last_active":"2024-08-17T17:44:01.149288+0000","last_peered":"2024-08-17T17:44:01.149288+0000","last_clean":"2024-08-17T17:44:01.149288+0000","last_became_active":"2024-08-17T17:39:46.550870+0000","last_became_peered":"2024-08-17T17:39:46.550870+0000","last_unstale":"2024-08-17T17:44:01.149288+0000","last_undegraded":"2024-08-17T17:44:01.149288+0000","last_fullsized":"2024-08-17T17:44:01.149288+0000","mapping_epoch":375,"log_start":"390'6200","ondisk_log_start":"390'6200","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9185,"log_dups_size":0,"ondisk_log_size":9185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:37.360078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15385,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":476,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.455080+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:43:48.455080+0000","last_peered":"2024-08-17T17:43:48.455080+0000","last_clean":"2024-08-17T17:43:48.455080+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:43:48.455080+0000","last_undegraded":"2024-08-17T17:43:48.455080+0000","last_fullsized":"2024-08-17T17:43:48.455080+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":786,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935260+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:43:29.935260+0000","last_peered":"2024-08-17T17:43:29.935260+0000","last_clean":"2024-08-17T17:43:29.935260+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:43:29.935260+0000","last_undegraded":"2024-08-17T17:43:29.935260+0000","last_fullsized":"2024-08-17T17:43:29.935260+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":680,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.477430+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:43:48.477430+0000","last_peered":"2024-08-17T17:43:48.477430+0000","last_clean":"2024-08-17T17:43:48.477430+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:43:48.477430+0000","last_undegraded":"2024-08-17T17:43:48.477430+0000","last_fullsized":"2024-08-17T17:43:48.477430+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":677,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.477043+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:43:48.477043+0000","last_peered":"2024-08-17T17:43:48.477043+0000","last_clean":"2024-08-17T17:43:48.477043+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:43:48.477043+0000","last_undegraded":"2024-08-17T17:43:48.477043+0000","last_fullsized":"2024-08-17T17:43:48.477043+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"406'15240","reported_seq":15281,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.209762+0000","last_change":"2024-08-17T17:39:46.536940+0000","last_active":"2024-08-17T17:44:01.209762+0000","last_peered":"2024-08-17T17:44:01.209762+0000","last_clean":"2024-08-17T17:44:01.209762+0000","last_became_active":"2024-08-17T17:39:46.536551+0000","last_became_peered":"2024-08-17T17:39:46.536551+0000","last_unstale":"2024-08-17T17:44:01.209762+0000","last_undegraded":"2024-08-17T17:44:01.209762+0000","last_fullsized":"2024-08-17T17:44:01.209762+0000","mapping_epoch":375,"log_start":"390'6400","ondisk_log_start":"390'6400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8840,"log_dups_size":0,"ondisk_log_size":8840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:18.571897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15240,"num_write_kb":496384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":471,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:35.910653+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:43:35.910653+0000","last_peered":"2024-08-17T17:43:35.910653+0000","last_clean":"2024-08-17T17:43:35.910653+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:43:35.910653+0000","last_undegraded":"2024-08-17T17:43:35.910653+0000","last_fullsized":"2024-08-17T17:43:35.910653+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":484,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465460+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:44:04.465460+0000","last_peered":"2024-08-17T17:44:04.465460+0000","last_clean":"2024-08-17T17:44:04.465460+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:44:04.465460+0000","last_undegraded":"2024-08-17T17:44:04.465460+0000","last_fullsized":"2024-08-17T17:44:04.465460+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":670,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.455185+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:43:48.455185+0000","last_peered":"2024-08-17T17:43:48.455185+0000","last_clean":"2024-08-17T17:43:48.455185+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:43:48.455185+0000","last_undegraded":"2024-08-17T17:43:48.455185+0000","last_fullsized":"2024-08-17T17:43:48.455185+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":717,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.454649+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:43:48.454649+0000","last_peered":"2024-08-17T17:43:48.454649+0000","last_clean":"2024-08-17T17:43:48.454649+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:43:48.454649+0000","last_undegraded":"2024-08-17T17:43:48.454649+0000","last_fullsized":"2024-08-17T17:43:48.454649+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"406'15282","reported_seq":15356,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.232140+0000","last_change":"2024-08-17T17:39:46.540167+0000","last_active":"2024-08-17T17:44:01.232140+0000","last_peered":"2024-08-17T17:44:01.232140+0000","last_clean":"2024-08-17T17:44:01.232140+0000","last_became_active":"2024-08-17T17:39:46.539837+0000","last_became_peered":"2024-08-17T17:39:46.539837+0000","last_unstale":"2024-08-17T17:44:01.232140+0000","last_undegraded":"2024-08-17T17:44:01.232140+0000","last_fullsized":"2024-08-17T17:44:01.232140+0000","mapping_epoch":375,"log_start":"389'6100","ondisk_log_start":"389'6100","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":9182,"log_dups_size":0,"ondisk_log_size":9182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:22.590119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18743296,"num_objects":286,"num_object_clones":0,"num_object_copies":858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15282,"num_write_kb":498176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":423,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451065+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:43:48.451065+0000","last_peered":"2024-08-17T17:43:48.451065+0000","last_clean":"2024-08-17T17:43:48.451065+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:43:48.451065+0000","last_undegraded":"2024-08-17T17:43:48.451065+0000","last_fullsized":"2024-08-17T17:43:48.451065+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1123,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.275527+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:43:46.275527+0000","last_peered":"2024-08-17T17:43:46.275527+0000","last_clean":"2024-08-17T17:43:46.275527+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:43:46.275527+0000","last_undegraded":"2024-08-17T17:43:46.275527+0000","last_fullsized":"2024-08-17T17:43:46.275527+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":495,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.461787+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:43:48.461787+0000","last_peered":"2024-08-17T17:43:48.461787+0000","last_clean":"2024-08-17T17:43:48.461787+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:43:48.461787+0000","last_undegraded":"2024-08-17T17:43:48.461787+0000","last_fullsized":"2024-08-17T17:43:48.461787+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:36.240686+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:43:36.240686+0000","last_peered":"2024-08-17T17:43:36.240686+0000","last_clean":"2024-08-17T17:43:36.240686+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:43:36.240686+0000","last_undegraded":"2024-08-17T17:43:36.240686+0000","last_fullsized":"2024-08-17T17:43:36.240686+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"406'15238","reported_seq":15279,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.272984+0000","last_change":"2024-08-17T17:39:46.535639+0000","last_active":"2024-08-17T17:44:01.272984+0000","last_peered":"2024-08-17T17:44:01.272984+0000","last_clean":"2024-08-17T17:44:01.272984+0000","last_became_active":"2024-08-17T17:39:46.535386+0000","last_became_peered":"2024-08-17T17:39:46.535386+0000","last_unstale":"2024-08-17T17:44:01.272984+0000","last_undegraded":"2024-08-17T17:44:01.272984+0000","last_fullsized":"2024-08-17T17:44:01.272984+0000","mapping_epoch":375,"log_start":"390'6400","ondisk_log_start":"390'6400","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:43.860203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17694720,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15238,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":475,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461135+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:44:04.461135+0000","last_peered":"2024-08-17T17:44:04.461135+0000","last_clean":"2024-08-17T17:44:04.461135+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:44:04.461135+0000","last_undegraded":"2024-08-17T17:44:04.461135+0000","last_fullsized":"2024-08-17T17:44:04.461135+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":192,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.275604+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:43:46.275604+0000","last_peered":"2024-08-17T17:43:46.275604+0000","last_clean":"2024-08-17T17:43:46.275604+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:43:46.275604+0000","last_undegraded":"2024-08-17T17:43:46.275604+0000","last_fullsized":"2024-08-17T17:43:46.275604+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":962,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.458832+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:43:48.458832+0000","last_peered":"2024-08-17T17:43:48.458832+0000","last_clean":"2024-08-17T17:43:48.458832+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:43:48.458832+0000","last_undegraded":"2024-08-17T17:43:48.458832+0000","last_fullsized":"2024-08-17T17:43:48.458832+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":686,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.455787+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:43:48.455787+0000","last_peered":"2024-08-17T17:43:48.455787+0000","last_clean":"2024-08-17T17:43:48.455787+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:43:48.455787+0000","last_undegraded":"2024-08-17T17:43:48.455787+0000","last_fullsized":"2024-08-17T17:43:48.455787+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"406'15485","reported_seq":15526,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:00.920998+0000","last_change":"2024-08-17T17:39:46.548505+0000","last_active":"2024-08-17T17:44:00.920998+0000","last_peered":"2024-08-17T17:44:00.920998+0000","last_clean":"2024-08-17T17:44:00.920998+0000","last_became_active":"2024-08-17T17:39:46.548096+0000","last_became_peered":"2024-08-17T17:39:46.548096+0000","last_unstale":"2024-08-17T17:44:00.920998+0000","last_undegraded":"2024-08-17T17:44:00.920998+0000","last_fullsized":"2024-08-17T17:44:00.920998+0000","mapping_epoch":375,"log_start":"396'8503","ondisk_log_start":"396'8503","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:56.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19070976,"num_objects":291,"num_object_clones":0,"num_object_copies":873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15485,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":479,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.472062+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:43:48.472062+0000","last_peered":"2024-08-17T17:43:48.472062+0000","last_clean":"2024-08-17T17:43:48.472062+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:43:48.472062+0000","last_undegraded":"2024-08-17T17:43:48.472062+0000","last_fullsized":"2024-08-17T17:43:48.472062+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":683,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.458741+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:43:48.458741+0000","last_peered":"2024-08-17T17:43:48.458741+0000","last_clean":"2024-08-17T17:43:48.458741+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:43:48.458741+0000","last_undegraded":"2024-08-17T17:43:48.458741+0000","last_fullsized":"2024-08-17T17:43:48.458741+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":975,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935955+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:43:29.935955+0000","last_peered":"2024-08-17T17:43:29.935955+0000","last_clean":"2024-08-17T17:43:29.935955+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:43:29.935955+0000","last_undegraded":"2024-08-17T17:43:29.935955+0000","last_fullsized":"2024-08-17T17:43:29.935955+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":680,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.476413+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:43:48.476413+0000","last_peered":"2024-08-17T17:43:48.476413+0000","last_clean":"2024-08-17T17:43:48.476413+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:43:48.476413+0000","last_undegraded":"2024-08-17T17:43:48.476413+0000","last_fullsized":"2024-08-17T17:43:48.476413+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"406'15468","reported_seq":15510,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:00.887529+0000","last_change":"2024-08-17T17:39:46.540048+0000","last_active":"2024-08-17T17:44:00.887529+0000","last_peered":"2024-08-17T17:44:00.887529+0000","last_clean":"2024-08-17T17:44:00.887529+0000","last_became_active":"2024-08-17T17:39:46.539567+0000","last_became_peered":"2024-08-17T17:39:46.539567+0000","last_unstale":"2024-08-17T17:44:00.887529+0000","last_undegraded":"2024-08-17T17:44:00.887529+0000","last_fullsized":"2024-08-17T17:44:00.887529+0000","mapping_epoch":375,"log_start":"396'8414","ondisk_log_start":"396'8414","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:48.938130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299224,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15468,"num_write_kb":505345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":423,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.462671+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:43:48.462671+0000","last_peered":"2024-08-17T17:43:48.462671+0000","last_clean":"2024-08-17T17:43:48.462671+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:43:48.462671+0000","last_undegraded":"2024-08-17T17:43:48.462671+0000","last_fullsized":"2024-08-17T17:43:48.462671+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":675,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461415+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:44:04.461415+0000","last_peered":"2024-08-17T17:44:04.461415+0000","last_clean":"2024-08-17T17:44:04.461415+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:44:04.461415+0000","last_undegraded":"2024-08-17T17:44:04.461415+0000","last_fullsized":"2024-08-17T17:44:04.461415+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1193,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.456285+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:43:48.456285+0000","last_peered":"2024-08-17T17:43:48.456285+0000","last_clean":"2024-08-17T17:43:48.456285+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:43:48.456285+0000","last_undegraded":"2024-08-17T17:43:48.456285+0000","last_fullsized":"2024-08-17T17:43:48.456285+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":673,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460464+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:44:04.460464+0000","last_peered":"2024-08-17T17:44:04.460464+0000","last_clean":"2024-08-17T17:44:04.460464+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:44:04.460464+0000","last_undegraded":"2024-08-17T17:44:04.460464+0000","last_fullsized":"2024-08-17T17:44:04.460464+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"406'15835","reported_seq":15911,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461596+0000","last_change":"2024-08-17T17:39:46.547441+0000","last_active":"2024-08-17T17:44:04.461596+0000","last_peered":"2024-08-17T17:44:04.461596+0000","last_clean":"2024-08-17T17:44:04.461596+0000","last_became_active":"2024-08-17T17:39:46.547217+0000","last_became_peered":"2024-08-17T17:39:46.547217+0000","last_unstale":"2024-08-17T17:44:04.461596+0000","last_undegraded":"2024-08-17T17:44:04.461596+0000","last_fullsized":"2024-08-17T17:44:04.461596+0000","mapping_epoch":375,"log_start":"397'8800","ondisk_log_start":"397'8800","created":375,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:39:45.520488+0000","last_clean_scrub_stamp":"2024-08-17T17:39:45.520488+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:29.304426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15835,"num_write_kb":510592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":475,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.461713+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:44:04.461713+0000","last_peered":"2024-08-17T17:44:04.461713+0000","last_clean":"2024-08-17T17:44:04.461713+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:44:04.461713+0000","last_undegraded":"2024-08-17T17:44:04.461713+0000","last_fullsized":"2024-08-17T17:44:04.461713+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":692,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.472118+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:43:48.472118+0000","last_peered":"2024-08-17T17:43:48.472118+0000","last_clean":"2024-08-17T17:43:48.472118+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:43:48.472118+0000","last_undegraded":"2024-08-17T17:43:48.472118+0000","last_fullsized":"2024-08-17T17:43:48.472118+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":39,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:45.931490+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:43:45.931490+0000","last_peered":"2024-08-17T17:43:45.931490+0000","last_clean":"2024-08-17T17:43:45.931490+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:43:45.931490+0000","last_undegraded":"2024-08-17T17:43:45.931490+0000","last_fullsized":"2024-08-17T17:43:45.931490+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":350,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936220+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:43:29.936220+0000","last_peered":"2024-08-17T17:43:29.936220+0000","last_clean":"2024-08-17T17:43:29.936220+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:43:29.936220+0000","last_undegraded":"2024-08-17T17:43:29.936220+0000","last_fullsized":"2024-08-17T17:43:29.936220+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":313,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936598+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:43:29.936598+0000","last_peered":"2024-08-17T17:43:29.936598+0000","last_clean":"2024-08-17T17:43:29.936598+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:43:29.936598+0000","last_undegraded":"2024-08-17T17:43:29.936598+0000","last_fullsized":"2024-08-17T17:43:29.936598+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":323,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936542+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:43:29.936542+0000","last_peered":"2024-08-17T17:43:29.936542+0000","last_clean":"2024-08-17T17:43:29.936542+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:43:29.936542+0000","last_undegraded":"2024-08-17T17:43:29.936542+0000","last_fullsized":"2024-08-17T17:43:29.936542+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":668,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.455664+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:43:48.455664+0000","last_peered":"2024-08-17T17:43:48.455664+0000","last_clean":"2024-08-17T17:43:48.455664+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:43:48.455664+0000","last_undegraded":"2024-08-17T17:43:48.455664+0000","last_fullsized":"2024-08-17T17:43:48.455664+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":407,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.935628+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:43:29.935628+0000","last_peered":"2024-08-17T17:43:29.935628+0000","last_clean":"2024-08-17T17:43:29.935628+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:43:29.935628+0000","last_undegraded":"2024-08-17T17:43:29.935628+0000","last_fullsized":"2024-08-17T17:43:29.935628+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":191,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.933612+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:43:29.933612+0000","last_peered":"2024-08-17T17:43:29.933612+0000","last_clean":"2024-08-17T17:43:29.933612+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:43:29.933612+0000","last_undegraded":"2024-08-17T17:43:29.933612+0000","last_fullsized":"2024-08-17T17:43:29.933612+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":637,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465102+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:44:04.465102+0000","last_peered":"2024-08-17T17:44:04.465102+0000","last_clean":"2024-08-17T17:44:04.465102+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:44:04.465102+0000","last_undegraded":"2024-08-17T17:44:04.465102+0000","last_fullsized":"2024-08-17T17:44:04.465102+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":694,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.474035+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:43:48.474035+0000","last_peered":"2024-08-17T17:43:48.474035+0000","last_clean":"2024-08-17T17:43:48.474035+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:43:48.474035+0000","last_undegraded":"2024-08-17T17:43:48.474035+0000","last_fullsized":"2024-08-17T17:43:48.474035+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":686,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460362+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:44:04.460362+0000","last_peered":"2024-08-17T17:44:04.460362+0000","last_clean":"2024-08-17T17:44:04.460362+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:44:04.460362+0000","last_undegraded":"2024-08-17T17:44:04.460362+0000","last_fullsized":"2024-08-17T17:44:04.460362+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.279083+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:43:46.279083+0000","last_peered":"2024-08-17T17:43:46.279083+0000","last_clean":"2024-08-17T17:43:46.279083+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:43:46.279083+0000","last_undegraded":"2024-08-17T17:43:46.279083+0000","last_fullsized":"2024-08-17T17:43:46.279083+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":691,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460672+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:44:04.460672+0000","last_peered":"2024-08-17T17:44:04.460672+0000","last_clean":"2024-08-17T17:44:04.460672+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:44:04.460672+0000","last_undegraded":"2024-08-17T17:44:04.460672+0000","last_fullsized":"2024-08-17T17:44:04.460672+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":667,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.476964+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:43:48.476964+0000","last_peered":"2024-08-17T17:43:48.476964+0000","last_clean":"2024-08-17T17:43:48.476964+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:43:48.476964+0000","last_undegraded":"2024-08-17T17:43:48.476964+0000","last_fullsized":"2024-08-17T17:43:48.476964+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":670,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.454640+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:43:48.454640+0000","last_peered":"2024-08-17T17:43:48.454640+0000","last_clean":"2024-08-17T17:43:48.454640+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:43:48.454640+0000","last_undegraded":"2024-08-17T17:43:48.454640+0000","last_fullsized":"2024-08-17T17:43:48.454640+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":688,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460934+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:44:04.460934+0000","last_peered":"2024-08-17T17:44:04.460934+0000","last_clean":"2024-08-17T17:44:04.460934+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:44:04.460934+0000","last_undegraded":"2024-08-17T17:44:04.460934+0000","last_fullsized":"2024-08-17T17:44:04.460934+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":297,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936269+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:43:29.936269+0000","last_peered":"2024-08-17T17:43:29.936269+0000","last_clean":"2024-08-17T17:43:29.936269+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:43:29.936269+0000","last_undegraded":"2024-08-17T17:43:29.936269+0000","last_fullsized":"2024-08-17T17:43:29.936269+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.458977+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:43:48.458977+0000","last_peered":"2024-08-17T17:43:48.458977+0000","last_clean":"2024-08-17T17:43:48.458977+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:43:48.458977+0000","last_undegraded":"2024-08-17T17:43:48.458977+0000","last_fullsized":"2024-08-17T17:43:48.458977+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":61,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.476189+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:43:48.476189+0000","last_peered":"2024-08-17T17:43:48.476189+0000","last_clean":"2024-08-17T17:43:48.476189+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:43:48.476189+0000","last_undegraded":"2024-08-17T17:43:48.476189+0000","last_fullsized":"2024-08-17T17:43:48.476189+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936539+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:43:29.936539+0000","last_peered":"2024-08-17T17:43:29.936539+0000","last_clean":"2024-08-17T17:43:29.936539+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:43:29.936539+0000","last_undegraded":"2024-08-17T17:43:29.936539+0000","last_fullsized":"2024-08-17T17:43:29.936539+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":195,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.275578+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:43:46.275578+0000","last_peered":"2024-08-17T17:43:46.275578+0000","last_clean":"2024-08-17T17:43:46.275578+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:43:46.275578+0000","last_undegraded":"2024-08-17T17:43:46.275578+0000","last_fullsized":"2024-08-17T17:43:46.275578+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":297,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936185+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:43:29.936185+0000","last_peered":"2024-08-17T17:43:29.936185+0000","last_clean":"2024-08-17T17:43:29.936185+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:43:29.936185+0000","last_undegraded":"2024-08-17T17:43:29.936185+0000","last_fullsized":"2024-08-17T17:43:29.936185+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":205,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:46.275621+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:43:46.275621+0000","last_peered":"2024-08-17T17:43:46.275621+0000","last_clean":"2024-08-17T17:43:46.275621+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:43:46.275621+0000","last_undegraded":"2024-08-17T17:43:46.275621+0000","last_fullsized":"2024-08-17T17:43:46.275621+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":693,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465509+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:44:04.465509+0000","last_peered":"2024-08-17T17:44:04.465509+0000","last_clean":"2024-08-17T17:44:04.465509+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:44:04.465509+0000","last_undegraded":"2024-08-17T17:44:04.465509+0000","last_fullsized":"2024-08-17T17:44:04.465509+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":694,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460628+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:44:04.460628+0000","last_peered":"2024-08-17T17:44:04.460628+0000","last_clean":"2024-08-17T17:44:04.460628+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:44:04.460628+0000","last_undegraded":"2024-08-17T17:44:04.460628+0000","last_fullsized":"2024-08-17T17:44:04.460628+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":312,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:43:29.936023+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:43:29.936023+0000","last_peered":"2024-08-17T17:43:29.936023+0000","last_clean":"2024-08-17T17:43:29.936023+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:43:29.936023+0000","last_undegraded":"2024-08-17T17:43:29.936023+0000","last_fullsized":"2024-08-17T17:43:29.936023+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":711,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.477149+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:43:48.477149+0000","last_peered":"2024-08-17T17:43:48.477149+0000","last_clean":"2024-08-17T17:43:48.477149+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:43:48.477149+0000","last_undegraded":"2024-08-17T17:43:48.477149+0000","last_fullsized":"2024-08-17T17:43:48.477149+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":257097752,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5497,"num_objects_misplaced":250,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":246062,"num_write_kb":7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1273442304,"data_stored":1273430088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126407,"ondisk_log_size":126407,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201766,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990632,"kb_used_data":183456,"kb_used_omap":1,"kb_used_meta":807166,"kb_avail":92754520,"statfs":{"total":95995035648,"available":94980628480,"internally_reserved":0,"allocated":187858944,"data_stored":185861384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":826538443},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922263,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440788,"kb_used_data":36428,"kb_used_omap":7,"kb_used_meta":404280,"kb_avail":93304364,"statfs":{"total":95995035648,"available":95543668736,"internally_reserved":0,"allocated":37302272,"data_stored":35324134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":413983414},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085786,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661276,"kb_used_data":168980,"kb_used_omap":7,"kb_used_meta":492280,"kb_avail":93083876,"statfs":{"total":95995035648,"available":95317889024,"internally_reserved":0,"allocated":173035520,"data_stored":171046491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":504095161},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986676,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807584,"kb_used_data":72600,"kb_used_omap":1,"kb_used_meta":734974,"kb_avail":92937568,"statfs":{"total":95995035648,"available":95168069632,"internally_reserved":0,"allocated":74342400,"data_stored":72352946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":752613830},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435568,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":987920,"kb_used_data":76040,"kb_used_omap":5,"kb_used_meta":911866,"kb_avail":92757232,"statfs":{"total":95995035648,"available":94983405568,"internally_reserved":0,"allocated":77864960,"data_stored":75887634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":933750857},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,8],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599094,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889232,"kb_used_data":59272,"kb_used_omap":7,"kb_used_meta":829944,"kb_avail":92855920,"statfs":{"total":95995035648,"available":95084462080,"internally_reserved":0,"allocated":60694528,"data_stored":58716551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":849862829},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762620,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1462856,"kb_used_data":668864,"kb_used_omap":7,"kb_used_meta":793848,"kb_avail":92282296,"statfs":{"total":95995035648,"available":94497071104,"internally_reserved":0,"allocated":684916736,"data_stored":682945561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":812900769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926146,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961364,"kb_used_data":81420,"kb_used_omap":7,"kb_used_meta":879928,"kb_avail":92783788,"statfs":{"total":95995035648,"available":95010598912,"internally_reserved":0,"allocated":83374080,"data_stored":81392901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":901046876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":72417280,"data_stored":72417280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":671289344,"data_stored":671285272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":54067200,"data_stored":54067200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":64618496,"data_stored":64618496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":63242240,"data_stored":63242240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":164040704,"data_stored":164036632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":23461888,"data_stored":23461888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":160305152,"data_stored":160301080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:08.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:08.462 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:08.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:08.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:08.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[104516]: from='client.36006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[104516]: pgmap v1301: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:08.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5644/2478 objects degraded (227.764%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:08.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[109246]: from='client.36006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:08.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[109246]: pgmap v1301: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:08.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5644/2478 objects degraded (227.764%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:08.708 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:08.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:08.831 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:08 smithi175 ceph-mon[101434]: from='client.36006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:08 smithi175 ceph-mon[101434]: pgmap v1301: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5644/2478 objects degraded (227.764%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:08.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:08.956 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:09.058 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:09.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:09.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:09.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:09.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:09.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:09.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:09.399 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:44:09.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:09.418 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:44:09.419 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:09.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:09.436 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:44:09.436 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:09.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:09.460 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:44:09.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:09.477 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:44:09.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:09.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:44:09.495 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:09.512 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:44:09.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:09.529 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:44:09.529 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:09.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:09.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:09.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:09.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3325187387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:09.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3325187387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:09.704 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:09.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:09.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:09.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3325187387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:09.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:09.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:09.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:10.056 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:10.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:10.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:10.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:10.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:10.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:10.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:10.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:10.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:10.577 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:10.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[104516]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[109246]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:10.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:10.703 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:10.805 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:10.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.826 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:10.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:10 smithi175 ceph-mon[101434]: pgmap v1302: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 54 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5644/2478 objects degraded (227.764%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:44:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:10.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:10.950 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:11.052 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.076 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:11.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:11.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:11.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:11.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:11.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:44:11.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:11.465 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:11.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:11.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:11.573 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:11.573 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:11.593 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:11.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:11.719 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:11.821 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:11.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:11.864 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:11.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:11.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:12.070 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:12.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:12.194 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:12.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:12.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:12.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:12.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.465 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:12.590 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:12 smithi064 ceph-mon[104516]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 5224/417 objects degraded (1252.758%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:12.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:12 smithi064 ceph-mon[109246]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 5224/417 objects degraded (1252.758%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:12.692 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:12.713 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:12.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:12.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.839 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:12 smithi175 ceph-mon[101434]: pgmap v1303: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 5224/417 objects degraded (1252.758%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:12.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:12.963 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:13.065 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:13.087 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:13.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:13.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:13.211 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:13.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:13.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:13.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:13.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:13.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:13.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:13.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:44:13.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:13.584 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:13.687 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:13.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:13.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:13 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 5224/417 objects degraded (1252.758%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:13.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:13 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 5224/417 objects degraded (1252.758%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:13.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:13.833 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:13.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:13 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 5224/417 objects degraded (1252.758%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:13.935 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:13.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.957 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:13.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:14.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:14.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:14.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:14.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:13] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.51.0" 2024-08-17T17:44:14.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:14.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:14.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:14.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:14.452 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:14.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:14.575 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:14.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:14.699 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:14.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:14.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:14.873 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:14 smithi064 ceph-mon[104516]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 5224/84 objects degraded (6219.048%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:14.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:14 smithi064 ceph-mon[109246]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 5224/84 objects degraded (6219.048%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:14 smithi175 ceph-mon[101434]: pgmap v1304: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 5224/84 objects degraded (6219.048%); 659 KiB/s, 52 objects/s recovering 2024-08-17T17:44:14.925 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:14.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:15.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:15.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:15.171 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:15.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:15.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:15.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:15.418 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:15.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:44:15.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:15.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:15.474 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:15.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:15.562 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:15.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:15.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:15.687 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:15.789 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:15.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:15.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:15.935 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:15.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:16.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:16.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:16.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:16.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:16.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:16.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:16.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:16.435 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:16.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:16.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:16 smithi064 ceph-mon[104516]: pgmap v1305: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 5224/84 objects degraded (6219.048%); 605 KiB/s, 48 objects/s recovering 2024-08-17T17:44:16.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:16 smithi064 ceph-mon[109246]: pgmap v1305: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 5224/84 objects degraded (6219.048%); 605 KiB/s, 48 objects/s recovering 2024-08-17T17:44:16.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:16.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:16.688 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:16.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:16.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:16.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:16.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:16 smithi175 ceph-mon[101434]: pgmap v1305: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 5224/84 objects degraded (6219.048%); 605 KiB/s, 48 objects/s recovering 2024-08-17T17:44:16.914 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:16.935 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:17.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:17.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:17.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:17.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:17.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:17.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:17.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:17.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:17.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:17.445 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:17.477 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' removed 2024-08-17T17:44:17.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:17.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:17.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:17.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:17 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:44:17.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:17 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:44:17.659 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:17.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:17.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:17.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:17.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:17.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:17.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:17 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:44:17.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:18.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.051 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:18.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:18.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:18.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:18.479 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:44:18.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.549 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[104516]: pgmap v1306: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 4720/84 objects degraded (5619.048%); 549 KiB/s, 85 objects/s recovering 2024-08-17T17:44:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:44:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[104516]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:44:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[109246]: pgmap v1306: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 4720/84 objects degraded (5619.048%); 549 KiB/s, 85 objects/s recovering 2024-08-17T17:44:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:44:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[109246]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:44:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:18 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:18.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:18.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:18.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:18 smithi175 ceph-mon[101434]: pgmap v1306: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 4720/84 objects degraded (5619.048%); 549 KiB/s, 85 objects/s recovering 2024-08-17T17:44:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/549697334' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:44:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:18 smithi175 ceph-mon[101434]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:44:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:18 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:44:18.922 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:18.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:19.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:19.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:19.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:19.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:19.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:19.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:19.293 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:19.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.418 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:19.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:19.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:19.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:19.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:19.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:19.777 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:19.800 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:19.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:19.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:19.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[104516]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:44:19.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:19.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:19.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:19 smithi064 ceph-mon[109246]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:44:19.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:19.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:19 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2625548198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:19 smithi175 ceph-mon[101434]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:44:19.924 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:19.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:20.047 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:20.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.177 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:20.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:20.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:20.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:20.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:20.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:20.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:20.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:20.802 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:20.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:20 smithi064 ceph-mon[104516]: pgmap v1309: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:20.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:20 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:44:20.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:20 smithi064 ceph-mon[109246]: pgmap v1309: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:20.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:20 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:44:20.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:20 smithi175 ceph-mon[101434]: pgmap v1309: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:20 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4720/84 objects degraded (5619.048%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:44:20.931 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:21.032 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:21.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:21.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:21.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:21.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:44:21.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:21.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:21.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:21.436 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:21.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:21.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:21.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:21.687 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:21.789 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:21.813 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:21.915 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.940 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:21.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:22.041 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:22.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:22.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:22.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:22.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:22.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:22.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:22.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:22.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:22.543 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:22.567 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:22.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:22.669 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:22.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:22 smithi064 ceph-mon[104516]: pgmap v1310: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:22.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:22 smithi064 ceph-mon[109246]: pgmap v1310: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:22.693 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:22.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:22.822 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:22 smithi175 ceph-mon[101434]: pgmap v1310: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:22.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:22.949 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:23.014 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:44:23.015 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:44:23.051 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:23.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:23.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:23.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:23.204 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:23.306 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:23.332 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:23.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:23.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:23.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:23.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.588 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:23.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:23.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:23.715 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:23.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:23 smithi064 ceph-mon[104516]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:23.755 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:23 smithi064 ceph-mon[109246]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:23.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:23.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:23.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:23 smithi175 ceph-mon[101434]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:23.943 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:23.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:24.071 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.095 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:23] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.51.0" 2024-08-17T17:44:24.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:24.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:24.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.476 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.578 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.603 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:24.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:24.833 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:24 smithi175 ceph-mon[101434]: pgmap v1311: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:24.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/435371347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:24 smithi064 ceph-mon[104516]: pgmap v1311: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/435371347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:24 smithi064 ceph-mon[109246]: pgmap v1311: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:24.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/435371347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:24.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:24.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:25.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:25.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:25.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:25.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:25.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:25.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:25.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:25.463 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:25.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:25.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:25.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:25.610 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:25.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:25.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:25.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:25.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.860 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:25.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:25.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:25.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:26.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:26.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:26.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:26.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:26.160 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:26.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:26.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:26.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:26.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:26.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:26.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:26.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:26 smithi175 ceph-mon[101434]: pgmap v1312: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:26.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[104516]: pgmap v1312: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:26.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:26 smithi064 ceph-mon[109246]: pgmap v1312: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:26.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:26.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:26.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:26.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:26.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:26.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:26.746 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:26.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:26.875 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:26.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:27.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:27.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:27.132 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:27.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:27.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:27.363 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:27.385 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:27.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:27.509 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:27.611 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:27.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:27.711 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:27.733 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:27.754 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:27.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:27.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:27.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:28.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:28.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:28.129 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:28.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:28.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:28.258 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:28.334 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:44:28.335 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:44:28.335 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-08-17T17:44:28.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-08-17T17:44:28.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:28.381 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:28.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:28.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:28.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:28.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:28.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:28.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:28 smithi175 ceph-mon[101434]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:28 smithi064 ceph-mon[104516]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:28.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:28 smithi064 ceph-mon[109246]: pgmap v1313: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:28.734 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:28.755 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:28.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:28.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:28.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:29.003 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:29.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:29.105 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:29.127 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:29.132 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1310,"stamp":"2024-08-17T17:44:21.338229+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":8394979,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2672,"ondisk_log_size":2672,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":285,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6484148,"kb_used_data":553376,"kb_used_omap":46,"kb_used_meta":5930641,"kb_avail":743477068,"statfs":{"total":767960285184,"available":761320517632,"internally_reserved":0,"allocated":566657024,"data_stored":550735462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":6072976627},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":37,"apply_latency_ms":37,"commit_latency_ns":37000000,"apply_latency_ns":37000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-5224,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-249986,"num_write_kb":-7999489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2575,"num_bytes_recovered":-141230080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001864"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":677,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.438429+0000","last_change":"2024-08-17T17:18:55.498387+0000","last_active":"2024-08-17T17:44:19.438429+0000","last_peered":"2024-08-17T17:44:19.438429+0000","last_clean":"2024-08-17T17:44:19.438429+0000","last_became_active":"2024-08-17T17:18:55.497652+0000","last_became_peered":"2024-08-17T17:18:55.497652+0000","last_unstale":"2024-08-17T17:44:19.438429+0000","last_undegraded":"2024-08-17T17:44:19.438429+0000","last_fullsized":"2024-08-17T17:44:19.438429+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:24.783012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":410,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866023+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:44:17.866023+0000","last_peered":"2024-08-17T17:44:17.866023+0000","last_clean":"2024-08-17T17:44:17.866023+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:44:17.866023+0000","last_undegraded":"2024-08-17T17:44:17.866023+0000","last_fullsized":"2024-08-17T17:44:17.866023+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451470+0000","last_change":"2024-08-17T17:31:17.592454+0000","last_active":"2024-08-17T17:43:48.451470+0000","last_peered":"2024-08-17T17:43:48.451470+0000","last_clean":"2024-08-17T17:43:48.451470+0000","last_became_active":"2024-08-17T17:31:17.592061+0000","last_became_peered":"2024-08-17T17:31:17.592061+0000","last_unstale":"2024-08-17T17:43:48.451470+0000","last_undegraded":"2024-08-17T17:43:48.451470+0000","last_fullsized":"2024-08-17T17:43:48.451470+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:29:43.305397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":671,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.410749+0000","last_change":"2024-08-17T17:18:55.493170+0000","last_active":"2024-08-17T17:44:20.410749+0000","last_peered":"2024-08-17T17:44:20.410749+0000","last_clean":"2024-08-17T17:44:20.410749+0000","last_became_active":"2024-08-17T17:18:55.492140+0000","last_became_peered":"2024-08-17T17:18:55.492140+0000","last_unstale":"2024-08-17T17:44:20.410749+0000","last_undegraded":"2024-08-17T17:44:20.410749+0000","last_fullsized":"2024-08-17T17:44:20.410749+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:26.333226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":671,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411214+0000","last_change":"2024-08-17T17:18:55.494162+0000","last_active":"2024-08-17T17:44:20.411214+0000","last_peered":"2024-08-17T17:44:20.411214+0000","last_clean":"2024-08-17T17:44:20.411214+0000","last_became_active":"2024-08-17T17:18:55.493830+0000","last_became_peered":"2024-08-17T17:18:55.493830+0000","last_unstale":"2024-08-17T17:44:20.411214+0000","last_undegraded":"2024-08-17T17:44:20.411214+0000","last_fullsized":"2024-08-17T17:44:20.411214+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:38.301426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":396,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866181+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:44:17.866181+0000","last_peered":"2024-08-17T17:44:17.866181+0000","last_clean":"2024-08-17T17:44:17.866181+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:44:17.866181+0000","last_undegraded":"2024-08-17T17:44:17.866181+0000","last_fullsized":"2024-08-17T17:44:17.866181+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":677,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.434590+0000","last_change":"2024-08-17T17:18:55.495660+0000","last_active":"2024-08-17T17:44:19.434590+0000","last_peered":"2024-08-17T17:44:19.434590+0000","last_clean":"2024-08-17T17:44:19.434590+0000","last_became_active":"2024-08-17T17:18:55.495243+0000","last_became_peered":"2024-08-17T17:18:55.495243+0000","last_unstale":"2024-08-17T17:44:19.434590+0000","last_undegraded":"2024-08-17T17:44:19.434590+0000","last_fullsized":"2024-08-17T17:44:19.434590+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:08.734651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":343,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.457886+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:44:04.457886+0000","last_peered":"2024-08-17T17:44:04.457886+0000","last_clean":"2024-08-17T17:44:04.457886+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:44:04.457886+0000","last_undegraded":"2024-08-17T17:44:04.457886+0000","last_fullsized":"2024-08-17T17:44:04.457886+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":406,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866880+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:44:17.866880+0000","last_peered":"2024-08-17T17:44:17.866880+0000","last_clean":"2024-08-17T17:44:17.866880+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:44:17.866880+0000","last_undegraded":"2024-08-17T17:44:17.866880+0000","last_fullsized":"2024-08-17T17:44:17.866880+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":203,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.445020+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:44:04.445020+0000","last_peered":"2024-08-17T17:44:04.445020+0000","last_clean":"2024-08-17T17:44:04.445020+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:44:04.445020+0000","last_undegraded":"2024-08-17T17:44:04.445020+0000","last_fullsized":"2024-08-17T17:44:04.445020+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":407,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866721+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:44:17.866721+0000","last_peered":"2024-08-17T17:44:17.866721+0000","last_clean":"2024-08-17T17:44:17.866721+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:44:17.866721+0000","last_undegraded":"2024-08-17T17:44:17.866721+0000","last_fullsized":"2024-08-17T17:44:17.866721+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":302,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631525+0000","last_change":"2024-08-17T17:22:21.225287+0000","last_active":"2024-08-17T17:44:17.631525+0000","last_peered":"2024-08-17T17:44:17.631525+0000","last_clean":"2024-08-17T17:44:17.631525+0000","last_became_active":"2024-08-17T17:22:21.224782+0000","last_became_peered":"2024-08-17T17:22:21.224782+0000","last_unstale":"2024-08-17T17:44:17.631525+0000","last_undegraded":"2024-08-17T17:44:17.631525+0000","last_fullsized":"2024-08-17T17:44:17.631525+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:38.683054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":424,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866653+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:44:17.866653+0000","last_peered":"2024-08-17T17:44:17.866653+0000","last_clean":"2024-08-17T17:44:17.866653+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:44:17.866653+0000","last_undegraded":"2024-08-17T17:44:17.866653+0000","last_fullsized":"2024-08-17T17:44:17.866653+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":323,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.457316+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:44:04.457316+0000","last_peered":"2024-08-17T17:44:04.457316+0000","last_clean":"2024-08-17T17:44:04.457316+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:44:04.457316+0000","last_undegraded":"2024-08-17T17:44:04.457316+0000","last_fullsized":"2024-08-17T17:44:04.457316+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":202,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.450934+0000","last_change":"2024-08-17T17:31:37.156309+0000","last_active":"2024-08-17T17:43:48.450934+0000","last_peered":"2024-08-17T17:43:48.450934+0000","last_clean":"2024-08-17T17:43:48.450934+0000","last_became_active":"2024-08-17T17:31:37.155992+0000","last_became_peered":"2024-08-17T17:31:37.155992+0000","last_unstale":"2024-08-17T17:43:48.450934+0000","last_undegraded":"2024-08-17T17:43:48.450934+0000","last_fullsized":"2024-08-17T17:43:48.450934+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:35.950368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":686,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411484+0000","last_change":"2024-08-17T17:22:21.217762+0000","last_active":"2024-08-17T17:44:20.411484+0000","last_peered":"2024-08-17T17:44:20.411484+0000","last_clean":"2024-08-17T17:44:20.411484+0000","last_became_active":"2024-08-17T17:22:21.217369+0000","last_became_peered":"2024-08-17T17:22:21.217369+0000","last_unstale":"2024-08-17T17:44:20.411484+0000","last_undegraded":"2024-08-17T17:44:20.411484+0000","last_fullsized":"2024-08-17T17:44:20.411484+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:11.311342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":765,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866067+0000","last_change":"2024-08-17T17:40:19.544340+0000","last_active":"2024-08-17T17:44:17.866067+0000","last_peered":"2024-08-17T17:44:17.866067+0000","last_clean":"2024-08-17T17:44:17.866067+0000","last_became_active":"2024-08-17T17:39:41.345178+0000","last_became_peered":"2024-08-17T17:39:41.345178+0000","last_unstale":"2024-08-17T17:44:17.866067+0000","last_undegraded":"2024-08-17T17:44:17.866067+0000","last_fullsized":"2024-08-17T17:44:17.866067+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:40.592021+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":699,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.434882+0000","last_change":"2024-08-17T17:22:21.240686+0000","last_active":"2024-08-17T17:44:19.434882+0000","last_peered":"2024-08-17T17:44:19.434882+0000","last_clean":"2024-08-17T17:44:19.434882+0000","last_became_active":"2024-08-17T17:22:21.209488+0000","last_became_peered":"2024-08-17T17:22:21.209488+0000","last_unstale":"2024-08-17T17:44:19.434882+0000","last_undegraded":"2024-08-17T17:44:19.434882+0000","last_fullsized":"2024-08-17T17:44:19.434882+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:38.715816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":325,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.632259+0000","last_change":"2024-08-17T17:39:26.978367+0000","last_active":"2024-08-17T17:44:17.632259+0000","last_peered":"2024-08-17T17:44:17.632259+0000","last_clean":"2024-08-17T17:44:17.632259+0000","last_became_active":"2024-08-17T17:39:26.978117+0000","last_became_peered":"2024-08-17T17:39:26.978117+0000","last_unstale":"2024-08-17T17:44:17.632259+0000","last_undegraded":"2024-08-17T17:44:17.632259+0000","last_fullsized":"2024-08-17T17:44:17.632259+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:02.717143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":424,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.296353+0000","last_change":"2024-08-17T17:40:26.249808+0000","last_active":"2024-08-17T17:44:01.296353+0000","last_peered":"2024-08-17T17:44:01.296353+0000","last_clean":"2024-08-17T17:44:01.296353+0000","last_became_active":"2024-08-17T17:39:41.346795+0000","last_became_peered":"2024-08-17T17:39:41.346795+0000","last_unstale":"2024-08-17T17:44:01.296353+0000","last_undegraded":"2024-08-17T17:44:01.296353+0000","last_fullsized":"2024-08-17T17:44:01.296353+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:11.404405+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":331,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631578+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:44:17.631578+0000","last_peered":"2024-08-17T17:44:17.631578+0000","last_clean":"2024-08-17T17:44:17.631578+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:44:17.631578+0000","last_undegraded":"2024-08-17T17:44:17.631578+0000","last_fullsized":"2024-08-17T17:44:17.631578+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":301,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631955+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:44:17.631955+0000","last_peered":"2024-08-17T17:44:17.631955+0000","last_clean":"2024-08-17T17:44:17.631955+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:44:17.631955+0000","last_undegraded":"2024-08-17T17:44:17.631955+0000","last_fullsized":"2024-08-17T17:44:17.631955+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":302,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631445+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:44:17.631445+0000","last_peered":"2024-08-17T17:44:17.631445+0000","last_clean":"2024-08-17T17:44:17.631445+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:44:17.631445+0000","last_undegraded":"2024-08-17T17:44:17.631445+0000","last_fullsized":"2024-08-17T17:44:17.631445+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":316,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.632255+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:44:17.632255+0000","last_peered":"2024-08-17T17:44:17.632255+0000","last_clean":"2024-08-17T17:44:17.632255+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:44:17.632255+0000","last_undegraded":"2024-08-17T17:44:17.632255+0000","last_fullsized":"2024-08-17T17:44:17.632255+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":424,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.296386+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:44:01.296386+0000","last_peered":"2024-08-17T17:44:01.296386+0000","last_clean":"2024-08-17T17:44:01.296386+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:44:01.296386+0000","last_undegraded":"2024-08-17T17:44:01.296386+0000","last_fullsized":"2024-08-17T17:44:01.296386+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":679,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.435170+0000","last_change":"2024-08-17T17:18:55.497924+0000","last_active":"2024-08-17T17:44:19.435170+0000","last_peered":"2024-08-17T17:44:19.435170+0000","last_clean":"2024-08-17T17:44:19.435170+0000","last_became_active":"2024-08-17T17:18:55.496991+0000","last_became_peered":"2024-08-17T17:18:55.496991+0000","last_unstale":"2024-08-17T17:44:19.435170+0000","last_undegraded":"2024-08-17T17:44:19.435170+0000","last_fullsized":"2024-08-17T17:44:19.435170+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:08.250321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":665,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.463769+0000","last_change":"2024-08-17T17:22:21.212596+0000","last_active":"2024-08-17T17:44:04.463769+0000","last_peered":"2024-08-17T17:44:04.463769+0000","last_clean":"2024-08-17T17:44:04.463769+0000","last_became_active":"2024-08-17T17:22:21.212334+0000","last_became_peered":"2024-08-17T17:22:21.212334+0000","last_unstale":"2024-08-17T17:44:04.463769+0000","last_undegraded":"2024-08-17T17:44:04.463769+0000","last_fullsized":"2024-08-17T17:44:04.463769+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:55.769986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":749,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.865608+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:44:17.865608+0000","last_peered":"2024-08-17T17:44:17.865608+0000","last_clean":"2024-08-17T17:44:17.865608+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:44:17.865608+0000","last_undegraded":"2024-08-17T17:44:17.865608+0000","last_fullsized":"2024-08-17T17:44:17.865608+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1001,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451232+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:43:48.451232+0000","last_peered":"2024-08-17T17:43:48.451232+0000","last_clean":"2024-08-17T17:43:48.451232+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:43:48.451232+0000","last_undegraded":"2024-08-17T17:43:48.451232+0000","last_fullsized":"2024-08-17T17:43:48.451232+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":663,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.435793+0000","last_change":"2024-08-17T17:31:37.162674+0000","last_active":"2024-08-17T17:44:19.435793+0000","last_peered":"2024-08-17T17:44:19.435793+0000","last_clean":"2024-08-17T17:44:19.435793+0000","last_became_active":"2024-08-17T17:31:37.162190+0000","last_became_peered":"2024-08-17T17:31:37.162190+0000","last_unstale":"2024-08-17T17:44:19.435793+0000","last_undegraded":"2024-08-17T17:44:19.435793+0000","last_fullsized":"2024-08-17T17:44:19.435793+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:18.712530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":316,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411353+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:44:20.411353+0000","last_peered":"2024-08-17T17:44:20.411353+0000","last_clean":"2024-08-17T17:44:20.411353+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:44:20.411353+0000","last_undegraded":"2024-08-17T17:44:20.411353+0000","last_fullsized":"2024-08-17T17:44:20.411353+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":478,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466374+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:44:04.466374+0000","last_peered":"2024-08-17T17:44:04.466374+0000","last_clean":"2024-08-17T17:44:04.466374+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:44:04.466374+0000","last_undegraded":"2024-08-17T17:44:04.466374+0000","last_fullsized":"2024-08-17T17:44:04.466374+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":789,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.865585+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:44:17.865585+0000","last_peered":"2024-08-17T17:44:17.865585+0000","last_clean":"2024-08-17T17:44:17.865585+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:44:17.865585+0000","last_undegraded":"2024-08-17T17:44:17.865585+0000","last_fullsized":"2024-08-17T17:44:17.865585+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":686,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.410933+0000","last_change":"2024-08-17T17:22:21.217309+0000","last_active":"2024-08-17T17:44:20.410933+0000","last_peered":"2024-08-17T17:44:20.410933+0000","last_clean":"2024-08-17T17:44:20.410933+0000","last_became_active":"2024-08-17T17:22:21.216938+0000","last_became_peered":"2024-08-17T17:22:21.216938+0000","last_unstale":"2024-08-17T17:44:20.410933+0000","last_undegraded":"2024-08-17T17:44:20.410933+0000","last_fullsized":"2024-08-17T17:44:20.410933+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:18:35.398822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":683,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411361+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:44:20.411361+0000","last_peered":"2024-08-17T17:44:20.411361+0000","last_clean":"2024-08-17T17:44:20.411361+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:44:20.411361+0000","last_undegraded":"2024-08-17T17:44:20.411361+0000","last_fullsized":"2024-08-17T17:44:20.411361+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":474,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.865657+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:44:17.865657+0000","last_peered":"2024-08-17T17:44:17.865657+0000","last_clean":"2024-08-17T17:44:17.865657+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:44:17.865657+0000","last_undegraded":"2024-08-17T17:44:17.865657+0000","last_fullsized":"2024-08-17T17:44:17.865657+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":486,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.437854+0000","last_change":"2024-08-17T17:39:52.568763+0000","last_active":"2024-08-17T17:44:19.437854+0000","last_peered":"2024-08-17T17:44:19.437854+0000","last_clean":"2024-08-17T17:44:19.437854+0000","last_became_active":"2024-08-17T17:39:52.568170+0000","last_became_peered":"2024-08-17T17:39:52.568170+0000","last_unstale":"2024-08-17T17:44:19.437854+0000","last_undegraded":"2024-08-17T17:44:19.437854+0000","last_fullsized":"2024-08-17T17:44:19.437854+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:46.154121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":672,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466098+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:44:04.466098+0000","last_peered":"2024-08-17T17:44:04.466098+0000","last_clean":"2024-08-17T17:44:04.466098+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:44:04.466098+0000","last_undegraded":"2024-08-17T17:44:04.466098+0000","last_fullsized":"2024-08-17T17:44:04.466098+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":719,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465435+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:44:04.465435+0000","last_peered":"2024-08-17T17:44:04.465435+0000","last_clean":"2024-08-17T17:44:04.465435+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:44:04.465435+0000","last_undegraded":"2024-08-17T17:44:04.465435+0000","last_fullsized":"2024-08-17T17:44:04.465435+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":424,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.290104+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:43:51.290104+0000","last_peered":"2024-08-17T17:43:51.290104+0000","last_clean":"2024-08-17T17:43:51.290104+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:43:51.290104+0000","last_undegraded":"2024-08-17T17:43:51.290104+0000","last_fullsized":"2024-08-17T17:43:51.290104+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1124,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.452315+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:43:48.452315+0000","last_peered":"2024-08-17T17:43:48.452315+0000","last_clean":"2024-08-17T17:43:48.452315+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:43:48.452315+0000","last_undegraded":"2024-08-17T17:43:48.452315+0000","last_fullsized":"2024-08-17T17:43:48.452315+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":497,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.460774+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:44:04.460774+0000","last_peered":"2024-08-17T17:44:04.460774+0000","last_clean":"2024-08-17T17:44:04.460774+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:44:04.460774+0000","last_undegraded":"2024-08-17T17:44:04.460774+0000","last_fullsized":"2024-08-17T17:44:04.460774+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.450714+0000","last_change":"2024-08-17T17:39:26.585330+0000","last_active":"2024-08-17T17:43:48.450714+0000","last_peered":"2024-08-17T17:43:48.450714+0000","last_clean":"2024-08-17T17:43:48.450714+0000","last_became_active":"2024-08-17T17:39:26.585068+0000","last_became_peered":"2024-08-17T17:39:26.585068+0000","last_unstale":"2024-08-17T17:43:48.450714+0000","last_undegraded":"2024-08-17T17:43:48.450714+0000","last_fullsized":"2024-08-17T17:43:48.450714+0000","mapping_epoch":364,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:59.729939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":477,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.434486+0000","last_change":"2024-08-17T17:40:21.741450+0000","last_active":"2024-08-17T17:44:19.434486+0000","last_peered":"2024-08-17T17:44:19.434486+0000","last_clean":"2024-08-17T17:44:19.434486+0000","last_became_active":"2024-08-17T17:39:41.345605+0000","last_became_peered":"2024-08-17T17:39:41.345605+0000","last_unstale":"2024-08-17T17:44:19.434486+0000","last_undegraded":"2024-08-17T17:44:19.434486+0000","last_fullsized":"2024-08-17T17:44:19.434486+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.741320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.741320+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:56.493597+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":193,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451389+0000","last_change":"2024-08-17T17:31:17.590674+0000","last_active":"2024-08-17T17:43:48.451389+0000","last_peered":"2024-08-17T17:43:48.451389+0000","last_clean":"2024-08-17T17:43:48.451389+0000","last_became_active":"2024-08-17T17:31:17.590351+0000","last_became_peered":"2024-08-17T17:31:17.590351+0000","last_unstale":"2024-08-17T17:43:48.451389+0000","last_undegraded":"2024-08-17T17:43:48.451389+0000","last_fullsized":"2024-08-17T17:43:48.451389+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:59.318119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":964,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466340+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:44:04.466340+0000","last_peered":"2024-08-17T17:44:04.466340+0000","last_clean":"2024-08-17T17:44:04.466340+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:44:04.466340+0000","last_undegraded":"2024-08-17T17:44:04.466340+0000","last_fullsized":"2024-08-17T17:44:04.466340+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":688,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466150+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:44:04.466150+0000","last_peered":"2024-08-17T17:44:04.466150+0000","last_clean":"2024-08-17T17:44:04.466150+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:44:04.466150+0000","last_undegraded":"2024-08-17T17:44:04.466150+0000","last_fullsized":"2024-08-17T17:44:04.466150+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":485,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.410677+0000","last_change":"2024-08-17T17:40:20.020783+0000","last_active":"2024-08-17T17:44:20.410677+0000","last_peered":"2024-08-17T17:44:20.410677+0000","last_clean":"2024-08-17T17:44:20.410677+0000","last_became_active":"2024-08-17T17:39:41.347399+0000","last_became_peered":"2024-08-17T17:39:41.347399+0000","last_unstale":"2024-08-17T17:44:20.410677+0000","last_undegraded":"2024-08-17T17:44:20.410677+0000","last_fullsized":"2024-08-17T17:44:20.410677+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:28.366646+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":685,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466084+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:44:04.466084+0000","last_peered":"2024-08-17T17:44:04.466084+0000","last_clean":"2024-08-17T17:44:04.466084+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:44:04.466084+0000","last_undegraded":"2024-08-17T17:44:04.466084+0000","last_fullsized":"2024-08-17T17:44:04.466084+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":979,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866735+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:44:17.866735+0000","last_peered":"2024-08-17T17:44:17.866735+0000","last_clean":"2024-08-17T17:44:17.866735+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:44:17.866735+0000","last_undegraded":"2024-08-17T17:44:17.866735+0000","last_fullsized":"2024-08-17T17:44:17.866735+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":686,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411349+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:44:20.411349+0000","last_peered":"2024-08-17T17:44:20.411349+0000","last_clean":"2024-08-17T17:44:20.411349+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:44:20.411349+0000","last_undegraded":"2024-08-17T17:44:20.411349+0000","last_fullsized":"2024-08-17T17:44:20.411349+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631720+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:44:17.631720+0000","last_peered":"2024-08-17T17:44:17.631720+0000","last_clean":"2024-08-17T17:44:17.631720+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:44:17.631720+0000","last_undegraded":"2024-08-17T17:44:17.631720+0000","last_fullsized":"2024-08-17T17:44:17.631720+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":677,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.434745+0000","last_change":"2024-08-17T17:18:55.498751+0000","last_active":"2024-08-17T17:44:19.434745+0000","last_peered":"2024-08-17T17:44:19.434745+0000","last_clean":"2024-08-17T17:44:19.434745+0000","last_became_active":"2024-08-17T17:18:55.498012+0000","last_became_peered":"2024-08-17T17:18:55.498012+0000","last_unstale":"2024-08-17T17:44:19.434745+0000","last_undegraded":"2024-08-17T17:44:19.434745+0000","last_fullsized":"2024-08-17T17:44:19.434745+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:08.444667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1195,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466038+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:44:04.466038+0000","last_peered":"2024-08-17T17:44:04.466038+0000","last_clean":"2024-08-17T17:44:04.466038+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:44:04.466038+0000","last_undegraded":"2024-08-17T17:44:04.466038+0000","last_fullsized":"2024-08-17T17:44:04.466038+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":675,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.434974+0000","last_change":"2024-08-17T17:18:55.497327+0000","last_active":"2024-08-17T17:44:19.434974+0000","last_peered":"2024-08-17T17:44:19.434974+0000","last_clean":"2024-08-17T17:44:19.434974+0000","last_became_active":"2024-08-17T17:18:55.496454+0000","last_became_peered":"2024-08-17T17:18:55.496454+0000","last_unstale":"2024-08-17T17:44:19.434974+0000","last_undegraded":"2024-08-17T17:44:19.434974+0000","last_fullsized":"2024-08-17T17:44:19.434974+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:29.294665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":477,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.437653+0000","last_change":"2024-08-17T17:40:20.757030+0000","last_active":"2024-08-17T17:44:19.437653+0000","last_peered":"2024-08-17T17:44:19.437653+0000","last_clean":"2024-08-17T17:44:19.437653+0000","last_became_active":"2024-08-17T17:39:41.344799+0000","last_became_peered":"2024-08-17T17:39:41.344799+0000","last_unstale":"2024-08-17T17:44:19.437653+0000","last_undegraded":"2024-08-17T17:44:19.437653+0000","last_fullsized":"2024-08-17T17:44:19.437653+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.756316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.756316+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:31.066892+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":698,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.410790+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:44:20.410790+0000","last_peered":"2024-08-17T17:44:20.410790+0000","last_clean":"2024-08-17T17:44:20.410790+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:44:20.410790+0000","last_undegraded":"2024-08-17T17:44:20.410790+0000","last_fullsized":"2024-08-17T17:44:20.410790+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":42,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866646+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:44:17.866646+0000","last_peered":"2024-08-17T17:44:17.866646+0000","last_clean":"2024-08-17T17:44:17.866646+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:44:17.866646+0000","last_undegraded":"2024-08-17T17:44:17.866646+0000","last_fullsized":"2024-08-17T17:44:17.866646+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":353,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631634+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:44:17.631634+0000","last_peered":"2024-08-17T17:44:17.631634+0000","last_clean":"2024-08-17T17:44:17.631634+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:44:17.631634+0000","last_undegraded":"2024-08-17T17:44:17.631634+0000","last_fullsized":"2024-08-17T17:44:17.631634+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":316,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631870+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:44:17.631870+0000","last_peered":"2024-08-17T17:44:17.631870+0000","last_clean":"2024-08-17T17:44:17.631870+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:44:17.631870+0000","last_undegraded":"2024-08-17T17:44:17.631870+0000","last_fullsized":"2024-08-17T17:44:17.631870+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":326,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.632203+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:44:17.632203+0000","last_peered":"2024-08-17T17:44:17.632203+0000","last_clean":"2024-08-17T17:44:17.632203+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:44:17.632203+0000","last_undegraded":"2024-08-17T17:44:17.632203+0000","last_fullsized":"2024-08-17T17:44:17.632203+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":670,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.465981+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:44:04.465981+0000","last_peered":"2024-08-17T17:44:04.465981+0000","last_clean":"2024-08-17T17:44:04.465981+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:44:04.465981+0000","last_undegraded":"2024-08-17T17:44:04.465981+0000","last_fullsized":"2024-08-17T17:44:04.465981+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":411,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.866497+0000","last_change":"2024-08-17T17:22:21.257365+0000","last_active":"2024-08-17T17:44:17.866497+0000","last_peered":"2024-08-17T17:44:17.866497+0000","last_clean":"2024-08-17T17:44:17.866497+0000","last_became_active":"2024-08-17T17:22:21.210891+0000","last_became_peered":"2024-08-17T17:22:21.210891+0000","last_unstale":"2024-08-17T17:44:17.866497+0000","last_undegraded":"2024-08-17T17:44:17.866497+0000","last_fullsized":"2024-08-17T17:44:17.866497+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:19.572845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":192,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451161+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:43:48.451161+0000","last_peered":"2024-08-17T17:43:48.451161+0000","last_clean":"2024-08-17T17:43:48.451161+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:43:48.451161+0000","last_undegraded":"2024-08-17T17:43:48.451161+0000","last_fullsized":"2024-08-17T17:43:48.451161+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":639,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.437609+0000","last_change":"2024-08-17T17:31:36.157375+0000","last_active":"2024-08-17T17:44:19.437609+0000","last_peered":"2024-08-17T17:44:19.437609+0000","last_clean":"2024-08-17T17:44:19.437609+0000","last_became_active":"2024-08-17T17:31:36.156958+0000","last_became_peered":"2024-08-17T17:31:36.156958+0000","last_unstale":"2024-08-17T17:44:19.437609+0000","last_undegraded":"2024-08-17T17:44:19.437609+0000","last_fullsized":"2024-08-17T17:44:19.437609+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:13.252629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":700,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411457+0000","last_change":"2024-08-17T17:31:17.595683+0000","last_active":"2024-08-17T17:44:20.411457+0000","last_peered":"2024-08-17T17:44:20.411457+0000","last_clean":"2024-08-17T17:44:20.411457+0000","last_became_active":"2024-08-17T17:31:17.595428+0000","last_became_peered":"2024-08-17T17:31:17.595428+0000","last_unstale":"2024-08-17T17:44:20.411457+0000","last_undegraded":"2024-08-17T17:44:20.411457+0000","last_fullsized":"2024-08-17T17:44:20.411457+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:42.360068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":688,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.433652+0000","last_change":"2024-08-17T17:31:36.160435+0000","last_active":"2024-08-17T17:44:19.433652+0000","last_peered":"2024-08-17T17:44:19.433652+0000","last_clean":"2024-08-17T17:44:19.433652+0000","last_became_active":"2024-08-17T17:31:36.160109+0000","last_became_peered":"2024-08-17T17:31:36.160109+0000","last_unstale":"2024-08-17T17:44:19.433652+0000","last_undegraded":"2024-08-17T17:44:19.433652+0000","last_fullsized":"2024-08-17T17:44:19.433652+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:27.294540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451544+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:43:48.451544+0000","last_peered":"2024-08-17T17:43:48.451544+0000","last_clean":"2024-08-17T17:43:48.451544+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:43:48.451544+0000","last_undegraded":"2024-08-17T17:43:48.451544+0000","last_fullsized":"2024-08-17T17:43:48.451544+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":693,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.435069+0000","last_change":"2024-08-17T17:18:55.493535+0000","last_active":"2024-08-17T17:44:19.435069+0000","last_peered":"2024-08-17T17:44:19.435069+0000","last_clean":"2024-08-17T17:44:19.435069+0000","last_became_active":"2024-08-17T17:18:55.493079+0000","last_became_peered":"2024-08-17T17:18:55.493079+0000","last_unstale":"2024-08-17T17:44:19.435069+0000","last_undegraded":"2024-08-17T17:44:19.435069+0000","last_fullsized":"2024-08-17T17:44:19.435069+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:46.472697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":673,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411264+0000","last_change":"2024-08-17T17:18:55.496875+0000","last_active":"2024-08-17T17:44:20.411264+0000","last_peered":"2024-08-17T17:44:20.411264+0000","last_clean":"2024-08-17T17:44:20.411264+0000","last_became_active":"2024-08-17T17:18:55.496495+0000","last_became_peered":"2024-08-17T17:18:55.496495+0000","last_unstale":"2024-08-17T17:44:20.411264+0000","last_undegraded":"2024-08-17T17:44:20.411264+0000","last_fullsized":"2024-08-17T17:44:20.411264+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:52.970369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":672,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.468403+0000","last_change":"2024-08-17T17:22:21.229948+0000","last_active":"2024-08-17T17:44:04.468403+0000","last_peered":"2024-08-17T17:44:04.468403+0000","last_clean":"2024-08-17T17:44:04.468403+0000","last_became_active":"2024-08-17T17:22:21.229597+0000","last_became_peered":"2024-08-17T17:22:21.229597+0000","last_unstale":"2024-08-17T17:44:04.468403+0000","last_undegraded":"2024-08-17T17:44:04.468403+0000","last_fullsized":"2024-08-17T17:44:04.468403+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:09.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":690,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.433441+0000","last_change":"2024-08-17T17:22:21.206496+0000","last_active":"2024-08-17T17:44:19.433441+0000","last_peered":"2024-08-17T17:44:19.433441+0000","last_clean":"2024-08-17T17:44:19.433441+0000","last_became_active":"2024-08-17T17:22:21.206098+0000","last_became_peered":"2024-08-17T17:22:21.206098+0000","last_unstale":"2024-08-17T17:44:19.433441+0000","last_undegraded":"2024-08-17T17:44:19.433441+0000","last_fullsized":"2024-08-17T17:44:19.433441+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:50.476082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":300,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631674+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:44:17.631674+0000","last_peered":"2024-08-17T17:44:17.631674+0000","last_clean":"2024-08-17T17:44:17.631674+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:44:17.631674+0000","last_undegraded":"2024-08-17T17:44:17.631674+0000","last_fullsized":"2024-08-17T17:44:17.631674+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":201,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:44:04.466685+0000","last_change":"2024-08-17T17:40:47.342200+0000","last_active":"2024-08-17T17:44:04.466685+0000","last_peered":"2024-08-17T17:44:04.466685+0000","last_clean":"2024-08-17T17:44:04.466685+0000","last_became_active":"2024-08-17T17:40:47.341815+0000","last_became_peered":"2024-08-17T17:40:47.341815+0000","last_unstale":"2024-08-17T17:44:04.466685+0000","last_undegraded":"2024-08-17T17:44:04.466685+0000","last_fullsized":"2024-08-17T17:44:04.466685+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:29.145526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":67,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411022+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:44:20.411022+0000","last_peered":"2024-08-17T17:44:20.411022+0000","last_clean":"2024-08-17T17:44:20.411022+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:44:20.411022+0000","last_undegraded":"2024-08-17T17:44:20.411022+0000","last_fullsized":"2024-08-17T17:44:20.411022+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":300,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631781+0000","last_change":"2024-08-17T17:22:21.224143+0000","last_active":"2024-08-17T17:44:17.631781+0000","last_peered":"2024-08-17T17:44:17.631781+0000","last_clean":"2024-08-17T17:44:17.631781+0000","last_became_active":"2024-08-17T17:22:21.223522+0000","last_became_peered":"2024-08-17T17:22:21.223522+0000","last_unstale":"2024-08-17T17:44:17.631781+0000","last_undegraded":"2024-08-17T17:44:17.631781+0000","last_fullsized":"2024-08-17T17:44:17.631781+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:45.614146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":196,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.450850+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:43:48.450850+0000","last_peered":"2024-08-17T17:43:48.450850+0000","last_clean":"2024-08-17T17:43:48.450850+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:43:48.450850+0000","last_undegraded":"2024-08-17T17:43:48.450850+0000","last_fullsized":"2024-08-17T17:43:48.450850+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631651+0000","last_change":"2024-08-17T17:22:21.214552+0000","last_active":"2024-08-17T17:44:17.631651+0000","last_peered":"2024-08-17T17:44:17.631651+0000","last_clean":"2024-08-17T17:44:17.631651+0000","last_became_active":"2024-08-17T17:22:21.214250+0000","last_became_peered":"2024-08-17T17:22:21.214250+0000","last_unstale":"2024-08-17T17:44:17.631651+0000","last_undegraded":"2024-08-17T17:44:17.631651+0000","last_fullsized":"2024-08-17T17:44:17.631651+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:13.467248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":206,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:43:48.451472+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:43:48.451472+0000","last_peered":"2024-08-17T17:43:48.451472+0000","last_clean":"2024-08-17T17:43:48.451472+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:43:48.451472+0000","last_undegraded":"2024-08-17T17:43:48.451472+0000","last_fullsized":"2024-08-17T17:43:48.451472+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":695,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.437960+0000","last_change":"2024-08-17T17:18:55.500315+0000","last_active":"2024-08-17T17:44:19.437960+0000","last_peered":"2024-08-17T17:44:19.437960+0000","last_clean":"2024-08-17T17:44:19.437960+0000","last_became_active":"2024-08-17T17:18:55.499859+0000","last_became_peered":"2024-08-17T17:18:55.499859+0000","last_unstale":"2024-08-17T17:44:19.437960+0000","last_undegraded":"2024-08-17T17:44:19.437960+0000","last_fullsized":"2024-08-17T17:44:19.437960+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:38.226889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":696,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:19.433899+0000","last_change":"2024-08-17T17:31:36.160040+0000","last_active":"2024-08-17T17:44:19.433899+0000","last_peered":"2024-08-17T17:44:19.433899+0000","last_clean":"2024-08-17T17:44:19.433899+0000","last_became_active":"2024-08-17T17:31:36.159689+0000","last_became_peered":"2024-08-17T17:31:36.159689+0000","last_unstale":"2024-08-17T17:44:19.433899+0000","last_undegraded":"2024-08-17T17:44:19.433899+0000","last_fullsized":"2024-08-17T17:44:19.433899+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:17.620391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":316,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:44:17.631440+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:44:17.631440+0000","last_peered":"2024-08-17T17:44:17.631440+0000","last_clean":"2024-08-17T17:44:17.631440+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:44:17.631440+0000","last_undegraded":"2024-08-17T17:44:17.631440+0000","last_fullsized":"2024-08-17T17:44:17.631440+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":717,"reported_epoch":409,"state":"active+clean","last_fresh":"2024-08-17T17:44:20.411252+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:44:20.411252+0000","last_peered":"2024-08-17T17:44:20.411252+0000","last_clean":"2024-08-17T17:44:20.411252+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:44:20.411252+0000","last_undegraded":"2024-08-17T17:44:20.411252+0000","last_fullsized":"2024-08-17T17:44:20.411252+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":6352,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201770,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856852,"kb_used_data":9476,"kb_used_omap":1,"kb_used_meta":847358,"kb_avail":92888300,"statfs":{"total":95995035648,"available":95117619200,"internally_reserved":0,"allocated":9703424,"data_stored":7693710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":867695051},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922266,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409332,"kb_used_data":5032,"kb_used_omap":7,"kb_used_meta":404280,"kb_avail":93335820,"statfs":{"total":95995035648,"available":95575879680,"internally_reserved":0,"allocated":5152768,"data_stored":3162710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":413983414},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085789,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":489700,"kb_used_data":5596,"kb_used_omap":7,"kb_used_meta":484088,"kb_avail":93255452,"statfs":{"total":95995035648,"available":95493582848,"internally_reserved":0,"allocated":5730304,"data_stored":3741435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":495706553},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986680,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":748224,"kb_used_data":9140,"kb_used_omap":1,"kb_used_meta":739070,"kb_avail":92996928,"statfs":{"total":95995035648,"available":95228854272,"internally_reserved":0,"allocated":9359360,"data_stored":7357986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":756808134},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435571,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921628,"kb_used_data":5588,"kb_used_omap":5,"kb_used_meta":916026,"kb_avail":92823524,"statfs":{"total":95995035648,"available":95051288576,"internally_reserved":0,"allocated":5722112,"data_stored":3740874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":938010697},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":22,"apply_latency_ms":22,"commit_latency_ns":22000000,"apply_latency_ns":22000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599097,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":857436,"kb_used_data":5268,"kb_used_omap":7,"kb_used_meta":852152,"kb_avail":92887716,"statfs":{"total":95995035648,"available":95117021184,"internally_reserved":0,"allocated":5394432,"data_stored":3412543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":872603821},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762623,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1307728,"kb_used_data":504136,"kb_used_omap":7,"kb_used_meta":803576,"kb_avail":92437424,"statfs":{"total":95995035648,"available":94655922176,"internally_reserved":0,"allocated":516235264,"data_stored":514264249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":822862241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926150,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893248,"kb_used_data":9140,"kb_used_omap":7,"kb_used_meta":884088,"kb_avail":92851904,"statfs":{"total":95995035648,"available":95080349696,"internally_reserved":0,"allocated":9359360,"data_stored":7361955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":905306716},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:29.136 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:44:29.136 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:44:29.136 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-17T17:44:29.136 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:44:29.137 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:44:29.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-17T17:44:29.229 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:29.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:29.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:29.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:29.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:29.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:29.601 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:29.615 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.624 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:29.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:29.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:29.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.876 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:29.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:29.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:29.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:30.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:30.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:30.123 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:30.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:30.225 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:30.246 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:30.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:30.294 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:30.330 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:30.348 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:30.371 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:30.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:30.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:30.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:30.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:30.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:30.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:30.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[104516]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[109246]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:30 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:30.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:30.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:44:30.818 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:30 smithi175 ceph-mon[101434]: pgmap v1314: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:30.818 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:30 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.819 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:30 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.819 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:30 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.819 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:30 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:30.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:30.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:30.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:30.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:31.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:31.119 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:31.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:31.243 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:31.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:31.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:31.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:31.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:31.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:31.615 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:31.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:31.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:31.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:31.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:31.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:31.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:31.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:32.059 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:44:32.060 INFO:teuthology.orchestra.run.smithi175.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:44:32.060 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:44:32.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:32.117 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:32.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:32.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:32.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:32.345 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:32.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:32.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:32.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:32.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:32.616 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:32.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:32 smithi175 ceph-mon[101434]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:32.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:32 smithi064 ceph-mon[104516]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:32.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:32 smithi064 ceph-mon[109246]: pgmap v1315: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:32.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:32.739 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:32.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:32.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:32.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:32.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:33.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:33.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:33.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:33.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:33.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:33.356 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:33.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:33.481 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:33.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:33.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:33.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:33.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:33.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:33.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:33.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:33.903 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:44:33.903 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:44:33.903 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:44:33.903 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:44:33.903 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:44:33.909 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:33.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:33.983 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:34.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.110 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:34.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:33] "GET /metrics HTTP/1.1" 200 34290 "" "Prometheus/2.51.0" 2024-08-17T17:44:34.210 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:34.234 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:34.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:34.362 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:34.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.487 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:34.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:34.588 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:34.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:34.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:34.732 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:34.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:34 smithi064 ceph-mon[104516]: pgmap v1316: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:34.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:34 smithi064 ceph-mon[109246]: pgmap v1316: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:34.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:44:34.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:34.855 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:34.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:34 smithi175 ceph-mon[101434]: pgmap v1316: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:34.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:34.979 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:35.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:35.102 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:35.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:35.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:35.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:35.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:35.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:35.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:35.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:35.599 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:35.603 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:44:35.603 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' created 2024-08-17T17:44:35.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:35.724 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:35.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:35.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/710685736' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4267020049' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/710685736' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4267020049' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:35 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:35.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/710685736' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4267020049' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:35 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:44:35.954 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.976 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:35.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:36.077 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:36.099 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:36.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:36.222 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:36.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:36.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:36.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:36.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:36.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:36.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:36.698 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:36.720 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:36.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:36.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:36 smithi175 ceph-mon[101434]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:36 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:36 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:36 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:44:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:36 smithi175 ceph-mon[101434]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:44:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[104516]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[104516]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[109246]: pgmap v1317: 83 pgs: 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:44:36.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:36 smithi064 ceph-mon[109246]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:44:36.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:36.972 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-08-17T17:44:37.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:37.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:37.222 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:37.347 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:37.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.471 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:37.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:37.595 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.692 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:44:37.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:44:37.693 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 6, 7, 4, 5] 2024-08-17T17:44:37.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd out 3 2024-08-17T17:44:37.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:37.720 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:37.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:37.822 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:37.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:37 smithi175 ceph-mon[101434]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:44:37.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:37 smithi064 ceph-mon[104516]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:44:37.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:37 smithi064 ceph-mon[109246]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:44:37.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:37.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:38.016 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:38.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:38.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:38.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:38.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:38.217 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:38.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:38.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:38.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:38.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:38.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:38.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:38.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:38.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:38.720 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:38.762 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:38.763 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:38.823 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:38.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:38.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[104516]: pgmap v1320: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:38.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:44:38.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[104516]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:44:38.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[109246]: pgmap v1320: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:38.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:44:38.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:38 smithi064 ceph-mon[109246]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:44:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:38 smithi175 ceph-mon[101434]: pgmap v1320: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:38 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:44:38.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:38 smithi175 ceph-mon[101434]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:44:38.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:38.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:39.075 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:39.101 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:39.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:39.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:39.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:39.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:44:39.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:44:39.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:39.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:39.304 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:44:39.304 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:39.321 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:44:39.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:39.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:39.338 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:44:39.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:39.349 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:39.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:39.356 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:44:39.356 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:39.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:44:39.373 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:39.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:44:39.391 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:39.451 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:39.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:39.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:44:39.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:39.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:39.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:39.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:39.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:39 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3982551729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:39.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:39 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3982551729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:39.833 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:39.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:39.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:39 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3982551729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:39.961 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:39.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:40.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:40.112 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:40.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:40.239 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:40.340 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:40.364 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:40.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:40.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:40.489 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:40.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:40.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:40.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:40.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[104516]: pgmap v1322: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:40.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:40.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[109246]: pgmap v1322: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:40.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:40.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:40.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:40.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:40.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:40.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:40.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:40 smithi175 ceph-mon[101434]: pgmap v1322: 99 pgs: 12 creating+peering, 4 unknown, 83 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:40.967 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.994 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:40.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:41.095 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:41.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:41.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:41.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:41.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:41.371 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:41.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:41.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:41.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:41.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:41.620 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:41.722 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:41.745 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:41.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:41.870 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:41.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:41.998 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:42.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:42.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:42.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:42.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:42.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:42.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:42.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:42.505 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:42.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:42.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:42.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:42.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:42.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:42.865 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:42.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:42 smithi175 ceph-mon[101434]: pgmap v1323: 99 pgs: 5 creating+peering, 94 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:42 smithi064 ceph-mon[104516]: pgmap v1323: 99 pgs: 5 creating+peering, 94 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:42.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:42 smithi064 ceph-mon[109246]: pgmap v1323: 99 pgs: 5 creating+peering, 94 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:42.993 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:43.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:43.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:44:43.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:43.273 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:43.375 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.403 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:43.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:43.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:43.779 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:43.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:43.880 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:43.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:43.994 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:43] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.51.0" 2024-08-17T17:44:43.994 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:44:43.707+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e412 definitely_dead 0 2024-08-17T17:44:44.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:44.031 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:44.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:44.156 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:44.258 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:44.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:44.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:44.407 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:44.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:44.532 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:44.632 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.3. 2024-08-17T17:44:44.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:44.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[104516]: pgmap v1324: 99 pgs: 99 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:44.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/148740500' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[104516]: Client client.admin marked osd.3 out, while it was still marked up 2024-08-17T17:44:44.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[109246]: pgmap v1324: 99 pgs: 99 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:44.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/148740500' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:44 smithi064 ceph-mon[109246]: Client client.admin marked osd.3 out, while it was still marked up 2024-08-17T17:44:44.663 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:44.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:44.788 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:44.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:44 smithi175 ceph-mon[101434]: pgmap v1324: 99 pgs: 99 active+clean; 4.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:44:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:44 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/148740500' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:44 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-08-17T17:44:44.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:44 smithi175 ceph-mon[101434]: Client client.admin marked osd.3 out, while it was still marked up 2024-08-17T17:44:44.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:45.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:45.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:45.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:45.162 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:45.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:45.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:45.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:45.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:45.537 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:45.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:45.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:45.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-08-17T17:44:45.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:45 smithi064 ceph-mon[104516]: osdmap e413: 8 total, 8 up, 7 in 2024-08-17T17:44:45.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-08-17T17:44:45.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:45 smithi064 ceph-mon[109246]: osdmap e413: 8 total, 8 up, 7 in 2024-08-17T17:44:45.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:45.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:45.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:45.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-08-17T17:44:45.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:45 smithi175 ceph-mon[101434]: osdmap e413: 8 total, 8 up, 7 in 2024-08-17T17:44:45.921 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:46.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.045 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:46.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:46.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:46.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:46.294 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:46.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:46.418 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:46.519 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:46.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:46.531 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:46.531 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:46.548 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:46.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:46.673 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:46.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:46.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:46.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: pgmap v1326: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: osdmap e414: 8 total, 8 up, 7 in 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:46.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: pgmap v1326: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: osdmap e414: 8 total, 8 up, 7 in 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:46.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:46.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: pgmap v1326: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:46.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: osdmap e414: 8 total, 8 up, 7 in 2024-08-17T17:44:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:46.923 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:46.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.049 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.187 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:47.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:44:47.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.430 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:47.558 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:47.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.684 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:47.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:47.785 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:47.811 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:47.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:47 smithi064 ceph-mon[104516]: osdmap e415: 8 total, 8 up, 7 in 2024-08-17T17:44:47.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:47 smithi064 ceph-mon[109246]: osdmap e415: 8 total, 8 up, 7 in 2024-08-17T17:44:47.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:47 smithi175 ceph-mon[101434]: osdmap e415: 8 total, 8 up, 7 in 2024-08-17T17:44:47.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:47.938 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:48.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.067 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:48.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:48.193 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:48.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:48.318 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:48.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:48.445 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:48.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:48.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:48.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:48.694 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:48.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:48 smithi175 ceph-mon[101434]: pgmap v1329: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 52 KiB/s, 2 objects/s recovering 2024-08-17T17:44:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:48 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:44:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:48 smithi175 ceph-mon[101434]: osdmap e416: 8 total, 8 up, 7 in 2024-08-17T17:44:48.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:48.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[109246]: pgmap v1329: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 52 KiB/s, 2 objects/s recovering 2024-08-17T17:44:48.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:44:48.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[109246]: osdmap e416: 8 total, 8 up, 7 in 2024-08-17T17:44:48.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[104516]: pgmap v1329: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 52 KiB/s, 2 objects/s recovering 2024-08-17T17:44:48.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:44:48.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:48 smithi064 ceph-mon[104516]: osdmap e416: 8 total, 8 up, 7 in 2024-08-17T17:44:48.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:49.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:49.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:49.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:49.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:49.194 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:49.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:49.295 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.322 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:49.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:49.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:49.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:49.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:49.547 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:49.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:49.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:49.697 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:49.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:49.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:49.821 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:49.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:49 smithi064 ceph-mon[104516]: osdmap e417: 8 total, 8 up, 7 in 2024-08-17T17:44:49.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:49 smithi064 ceph-mon[109246]: osdmap e417: 8 total, 8 up, 7 in 2024-08-17T17:44:49.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:49.950 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:50.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:49 smithi175 ceph-mon[101434]: osdmap e417: 8 total, 8 up, 7 in 2024-08-17T17:44:50.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:50.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.303 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:50.328 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:50.393 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:50.431 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:50.455 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:50.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:50.669 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-08-17T17:44:50.669 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:44:50.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:50.707 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:50.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:50.936 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[104516]: pgmap v1332: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 3 objects/s recovering 2024-08-17T17:44:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/389570334' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1522402603' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[109246]: pgmap v1332: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 3 objects/s recovering 2024-08-17T17:44:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/389570334' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1522402603' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:50.963 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:51.064 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:51.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:51.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:50 smithi175 ceph-mon[101434]: pgmap v1332: 99 pgs: 5 remapped+peering, 1 peering, 93 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 38 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 3 objects/s recovering 2024-08-17T17:44:51.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/389570334' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:51.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:51.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1522402603' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:51.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:44:51.191 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:51.217 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:51.306 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:44:51.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:44:51.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:51.344 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:51.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:51.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:51.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:51.592 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:51.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:51.694 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:51.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.721 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:51.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:51.767 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:44:51.767 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:51.784 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:44:51.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:51.801 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:44:51.801 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:51.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:44:51.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:51.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:51.835 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:44:51.835 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:51.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:51.857 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:44:51.857 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:51.874 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:44:51.875 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:51.892 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:44:51.892 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:44:51.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[104516]: osdmap e418: 8 total, 8 up, 7 in 2024-08-17T17:44:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:44:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:51 smithi064 ceph-mon[109246]: osdmap e418: 8 total, 8 up, 7 in 2024-08-17T17:44:51.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:51.974 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:52.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:52.098 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:51 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:44:52.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:51 smithi175 ceph-mon[101434]: osdmap e418: 8 total, 8 up, 7 in 2024-08-17T17:44:52.199 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:52.221 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:52.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:52.346 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:52.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:52.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:52.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:52.653 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:52.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:52.777 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:52.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:52.900 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:52.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[104516]: 1.4 scrub starts 2024-08-17T17:44:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[104516]: 1.4 scrub ok 2024-08-17T17:44:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[104516]: pgmap v1334: 99 pgs: 2 remapped+peering, 97 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:44:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1149299145' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[109246]: 1.4 scrub starts 2024-08-17T17:44:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[109246]: 1.4 scrub ok 2024-08-17T17:44:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[109246]: pgmap v1334: 99 pgs: 2 remapped+peering, 97 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:44:52.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1149299145' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:53.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:53.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:53.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:53.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:52 smithi175 ceph-mon[101434]: 1.4 scrub starts 2024-08-17T17:44:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:52 smithi175 ceph-mon[101434]: 1.4 scrub ok 2024-08-17T17:44:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:52 smithi175 ceph-mon[101434]: pgmap v1334: 99 pgs: 2 remapped+peering, 97 active+clean; 4.4 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2024-08-17T17:44:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:52 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:44:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1149299145' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:53.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:53.276 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:53.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:53.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:53.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:53.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:53.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:53.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:53.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:53.778 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:53.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:53.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:53.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[109246]: 1.7 deep-scrub starts 2024-08-17T17:44:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[109246]: 1.7 deep-scrub ok 2024-08-17T17:44:53.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[109246]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:44:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[104516]: 1.7 deep-scrub starts 2024-08-17T17:44:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[104516]: 1.7 deep-scrub ok 2024-08-17T17:44:53.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-mon[104516]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:44:54.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:54.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:54.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:54.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:54.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:54.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:53 smithi175 ceph-mon[101434]: 1.7 deep-scrub starts 2024-08-17T17:44:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:53 smithi175 ceph-mon[101434]: 1.7 deep-scrub ok 2024-08-17T17:44:54.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:53 smithi175 ceph-mon[101434]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:44:54.254 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:54.280 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:54.281 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:44:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:44:53] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.51.0" 2024-08-17T17:44:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:54.382 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:54.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:44:54.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:54.510 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:54.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:54.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:54.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:54.732 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-08-17T17:44:54.734 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:44:54.734 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:44:54.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:54.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:54.790 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:54.794 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:54.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2024-08-17T17:44:54.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:44:54.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_306376 2024-08-17T17:44:54.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:44:54.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:44:54.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:54.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:54.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[104516]: pgmap v1335: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-08-17T17:44:54.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:44:54.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:44:54.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[109246]: pgmap v1335: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-08-17T17:44:54.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:44:54.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:54 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:44:55.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:44:55.025 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:55.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:55.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:55.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:54 smithi175 ceph-mon[101434]: pgmap v1335: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-08-17T17:44:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:54 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:44:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:54 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:44:55.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:55.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:55.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:55.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:55.442 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.480 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:55.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:55.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:55.618 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:44:55.618 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:44:55.618 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2024-08-17T17:44:55.618 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:44:55.619 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:55.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:55.756 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:55.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3925 3909 244.299 244.312 0.00232517 0.00407671 2024-08-17T17:44:55.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:55.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:55.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:56.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:56.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:56.138 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:56.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:56.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[104516]: pgmap v1336: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 KiB/s, 0 objects/s recovering 2024-08-17T17:44:56.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:56.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:56 smithi064 ceph-mon[109246]: pgmap v1336: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 KiB/s, 0 objects/s recovering 2024-08-17T17:44:56.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:56.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.314 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:56.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:56.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:44:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:56 smithi175 ceph-mon[101434]: pgmap v1336: 99 pgs: 99 active+clean; 4.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 KiB/s, 0 objects/s recovering 2024-08-17T17:44:56.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:56.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:56.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:56.591 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:56.693 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:56.731 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:56.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 15 7663 7648 238.981 233.688 0.00242189 0.00417746 2024-08-17T17:44:56.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:56.868 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:56.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:57.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:57.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:57.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:57.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:44:57.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:57.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:57.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:57.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:57.527 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:57.563 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:57.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:57.705 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:57.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 11039 11023 229.627 210.938 0.00425354 0.00434896 2024-08-17T17:44:57.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:57.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:57.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:57.985 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:58.087 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:58.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:58.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:58.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:58.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:58.400 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:58.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:58.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:58.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:44:58 smithi064 ceph-mon[104516]: pgmap v1337: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s 2024-08-17T17:44:58.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:44:58 smithi064 ceph-mon[109246]: pgmap v1337: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s 2024-08-17T17:44:58.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:44:58 smithi175 ceph-mon[101434]: pgmap v1337: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s 2024-08-17T17:44:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:58.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:58.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:58.797 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:58.798 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:58.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 14201 14185 221.614 197.625 0.0043649 0.00450502 2024-08-17T17:44:58.826 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:58.928 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:58.966 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:59.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:59.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:59.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:59.204 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.244 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:59.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:59.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:59.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:59.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:59.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:59.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:59.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:59.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:44:59.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:59.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 16667 16651 208.114 154.125 0.00524596 0.00479845 2024-08-17T17:44:59.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:59.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:44:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:00.013 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:00.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:00.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:00.188 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:00.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:00.326 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:00.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:00.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:00.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:00 smithi064 ceph-mon[104516]: pgmap v1338: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 438 op/s 2024-08-17T17:45:00.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:00 smithi064 ceph-mon[109246]: pgmap v1338: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 438 op/s 2024-08-17T17:45:00.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:00.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:00.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:00.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:00.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:00.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 18944 18928 197.145 142.312 0.00165796 0.00505907 2024-08-17T17:45:00.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:00.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:00.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:00 smithi175 ceph-mon[101434]: pgmap v1338: 99 pgs: 99 active+clean; 281 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 28 MiB/s wr, 438 op/s 2024-08-17T17:45:00.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:01.005 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:01.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:01.161 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:01.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:01.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:01.397 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:01.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:01.438 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:45:01.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:01.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:01.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3232205874' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:01 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3232205874' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:01 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:01.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:01.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 21012 20996 187.444 129.25 0.00459967 0.00532861 2024-08-17T17:45:01.829 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3232205874' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:01 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:45:01.931 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:01.965 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:02.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:02.104 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:02.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:02.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:02.243 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:02.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:02.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:02.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:02.454 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:45:02.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:02.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:02.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:02.658 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[104516]: pgmap v1339: 99 pgs: 99 active+clean; 872 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:45:02.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[104516]: osdmap e419: 8 total, 8 up, 7 in 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/664287368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[109246]: pgmap v1339: 99 pgs: 99 active+clean; 872 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[109246]: osdmap e419: 8 total, 8 up, 7 in 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/664287368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:02 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:02.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 23465 23449 183.176 153.312 0.00232802 0.00545222 2024-08-17T17:45:02.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:02.906 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:02 smithi175 ceph-mon[101434]: pgmap v1339: 99 pgs: 99 active+clean; 872 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:45:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:02 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:45:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:02 smithi175 ceph-mon[101434]: osdmap e419: 8 total, 8 up, 7 in 2024-08-17T17:45:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/664287368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:02 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:02.941 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:02.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:03.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:03.078 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:03.181 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:03.217 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:03.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:03.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:03.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:03.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:03.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:03.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:03.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:03.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:03.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[104516]: osdmap e420: 8 total, 8 up, 7 in 2024-08-17T17:45:03.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:03.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:03.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-mon[109246]: osdmap e420: 8 total, 8 up, 7 in 2024-08-17T17:45:03.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:03.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:03.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:03.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 25820 25804 179.175 147.188 0.00666451 0.00557508 2024-08-17T17:45:03.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:03.906 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:03.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:03 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:03 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:03.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:03 smithi175 ceph-mon[101434]: osdmap e420: 8 total, 8 up, 7 in 2024-08-17T17:45:04.008 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:04.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:04.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:04.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:04.205 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:03] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.51.0" 2024-08-17T17:45:04.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:04.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.323 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:04.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:04.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:04.459 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:04.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:04.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:04.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:04.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:04.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:04.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 28255 28239 176.475 152.188 0.00313078 0.00565184 2024-08-17T17:45:04.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:04.865 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:04 smithi175 ceph-mon[101434]: pgmap v1342: 99 pgs: 99 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:45:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:04 smithi064 ceph-mon[104516]: pgmap v1342: 99 pgs: 99 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:45:04.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:04 smithi064 ceph-mon[109246]: pgmap v1342: 99 pgs: 99 active+clean; 1.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:45:04.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:05.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:05.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:05.127 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:05.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:05.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:05.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:05.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:05.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.539 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:05.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:05.676 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:05.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:05.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 30244 30228 171.732 124.312 0.00669386 0.00581728 2024-08-17T17:45:05.809 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:05.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:05.943 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:06.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:06.068 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:06.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:06.170 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:06.206 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:06.308 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:06.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:06.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:06.484 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:06.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:06.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:06.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:06.767 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:06.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:06 smithi064 ceph-mon[104516]: pgmap v1343: 99 pgs: 99 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-17T17:45:06.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:06 smithi064 ceph-mon[109246]: pgmap v1343: 99 pgs: 99 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-17T17:45:06.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 32343 32327 168.341 131.188 0.00723224 0.00593423 2024-08-17T17:45:06.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:06.904 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:06 smithi175 ceph-mon[101434]: pgmap v1343: 99 pgs: 99 active+clean; 1.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-17T17:45:07.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:07.048 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:07.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:07.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:45:07.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:07.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:07.432 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:07.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:07.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:07.706 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:07.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:07.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 34504 34488 165.781 135.062 0.00608023 0.00602666 2024-08-17T17:45:07.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:07.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:07.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:08.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:08.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:08.151 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:08.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:08.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.295 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:08.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:08.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:45:08.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:08.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:08.419 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:45:08.419 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:08.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:08.452 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:45:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:08.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:45:08.480 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:08.513 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:45:08.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:08.535 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:08.546 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:45:08.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:08.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:08.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:45:08.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:08.609 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:45:08.610 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:08.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:08 smithi064 ceph-mon[104516]: pgmap v1344: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-08-17T17:45:08.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1642529703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:08 smithi064 ceph-mon[109246]: pgmap v1344: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-08-17T17:45:08.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1642529703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.714 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:08.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:08.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 36656 36640 163.546 134.5 0.0106734 0.00610877 2024-08-17T17:45:08.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:08.850 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:08.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:08 smithi175 ceph-mon[101434]: pgmap v1344: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-08-17T17:45:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1642529703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:08.953 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:08.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:08.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:09.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:09.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:09.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:09.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:09.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:09.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:09.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:09.546 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:09.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:09.670 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:09.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:09.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 15 38684 38669 161.096 126.812 0.0103838 0.00620267 2024-08-17T17:45:09.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:09.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:09.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:10.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:10.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:10.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:45:10.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:10.223 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:10.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:10.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:10.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:10.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:10.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:10.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:10.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:10 smithi175 ceph-mon[101434]: pgmap v1345: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.07k op/s 2024-08-17T17:45:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:45:10.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:10.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:10.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:10.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:10.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 40645 40629 158.683 122.5 0.00742801 0.00629623 2024-08-17T17:45:10.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:10.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[104516]: pgmap v1345: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.07k op/s 2024-08-17T17:45:10.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:45:10.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:10.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[109246]: pgmap v1345: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 130 MiB/s wr, 2.07k op/s 2024-08-17T17:45:10.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:45:10.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:10.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:10.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:11.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:11.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:11.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:11.169 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:11.169 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:11.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:11.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:11.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:11.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:11.478 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:11.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:11.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:11.716 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:11.751 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:11.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 42469 42453 156.055 114 0.00995653 0.00640304 2024-08-17T17:45:11.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:11.889 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:11.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:45:11.987 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:11.991 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:11.992 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:12.028 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:12.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:12.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.176 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:12.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:12.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:12.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:12.413 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:12.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:12.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:12.552 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:12.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:12.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:12.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:12.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:12.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:12.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 44415 44399 154.141 121.625 0.00436028 0.00648282 2024-08-17T17:45:12.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:12 smithi064 ceph-mon[104516]: pgmap v1346: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:45:12.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:12 smithi064 ceph-mon[109246]: pgmap v1346: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:45:12.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:12.855 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:12.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:12 smithi175 ceph-mon[101434]: pgmap v1346: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:45:12.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:45:12.957 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:13.000 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:13.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.135 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:13.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:13.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:13.270 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:13.372 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:13.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:13.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:13.542 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:13.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:13.671 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:13.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:13.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 46066 46050 151.459 103.188 0.00778769 0.00659745 2024-08-17T17:45:13.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:13.913 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:13.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:14.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:14.086 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:14.187 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:14.229 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:14.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:14.366 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:14.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:13] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-08-17T17:45:14.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:14.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:14.603 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:14.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:14.773 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:14.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:45:14.803782+0000 min lat: 0.00139059 max lat: 0.129535 avg lat: 0.00667637 2024-08-17T17:45:14.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:45:14.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 47917 47901 149.67 115.688 0.0110824 0.00667637 2024-08-17T17:45:14.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:14 smithi064 ceph-mon[104516]: pgmap v1347: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-08-17T17:45:14.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:14 smithi064 ceph-mon[109246]: pgmap v1347: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-08-17T17:45:14.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:14.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:14 smithi175 ceph-mon[101434]: pgmap v1347: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-08-17T17:45:14.916 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:15.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:15.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:15.158 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:15.178 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:15.280 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:15.318 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:15.420 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:15.554 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:15.577 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:15.679 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:15.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:15.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 49489 49473 147.221 98.25 0.00794663 0.00678776 2024-08-17T17:45:15.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:15.853 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:15.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:15.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:16.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:16.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:16.231 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:16.257 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:16.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:16.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:16.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:16.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:16.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.540 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:16.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:16.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:16.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:16.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:16 smithi064 ceph-mon[104516]: pgmap v1348: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:45:16.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:16 smithi064 ceph-mon[109246]: pgmap v1348: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:45:16.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:16.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 51276 51260 145.606 111.688 0.0117215 0.00686222 2024-08-17T17:45:16.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:16 smithi175 ceph-mon[101434]: pgmap v1348: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:45:16.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:16.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:17.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:17.105 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:17.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:17.206 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.240 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.342 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:17.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:17.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:17.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:17.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:17.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.760 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:17.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 53002 52986 143.965 107.875 0.0107181 0.00694141 2024-08-17T17:45:17.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.939 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:17.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:18.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:18.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:18.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:18.213 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:18.223 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:18.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:18.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.351 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:18.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:18.452 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:18.489 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:18.500 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:45:18.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:18.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:18 smithi064 ceph-mon[104516]: pgmap v1349: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:45:18.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:18.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:18 smithi064 ceph-mon[109246]: pgmap v1349: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:45:18.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:18.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:18.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:18.771 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:18.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:18.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 54637 54621 142.224 102.188 0.00630445 0.00702533 2024-08-17T17:45:18.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:18.909 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:18 smithi175 ceph-mon[101434]: pgmap v1349: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:45:18.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:19.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:19.050 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:19.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:19.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:19.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:19.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:19.332 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:19.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:19.472 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:19.520 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:45:19.575 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:19.609 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:19.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:19.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:19.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[104516]: osdmap e421: 8 total, 8 up, 7 in 2024-08-17T17:45:19.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3730790186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:19.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:19.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[109246]: osdmap e421: 8 total, 8 up, 7 in 2024-08-17T17:45:19.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3730790186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:19 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.711 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:19.746 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:19.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 56293 56277 140.675 103.5 0.00368508 0.00710302 2024-08-17T17:45:19.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:19.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:19.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:19.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:19 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:19.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2470607056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:19 smithi175 ceph-mon[101434]: osdmap e421: 8 total, 8 up, 7 in 2024-08-17T17:45:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3730790186' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:19 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:45:19.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:20.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:20.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:20.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:20.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:20.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:20.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:20.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:20.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:20.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:20.534 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:20.573 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:20.675 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:20.712 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:20.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 57896 57880 139.117 100.188 0.0126169 0.00718318 2024-08-17T17:45:20.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:20.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:20.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:20 smithi175 ceph-mon[101434]: pgmap v1351: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:45:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:20 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:45:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:20 smithi175 ceph-mon[101434]: osdmap e422: 8 total, 8 up, 7 in 2024-08-17T17:45:20.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[104516]: pgmap v1351: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:45:20.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:45:20.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[104516]: osdmap e422: 8 total, 8 up, 7 in 2024-08-17T17:45:20.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[109246]: pgmap v1351: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:45:20.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:45:20.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:20 smithi064 ceph-mon[109246]: osdmap e422: 8 total, 8 up, 7 in 2024-08-17T17:45:20.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:20.985 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:20.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:21.087 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:21.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:21.354 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.374 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:45:21.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:21.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:21.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:21.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:21.786 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:21.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 59281 59265 137.17 86.5625 0.0105219 0.00728576 2024-08-17T17:45:21.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.922 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:22.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:22.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:22.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:22.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:22.199 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:22.255 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:22.256 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:22.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:22.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:22.435 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:22.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:22.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:22.608 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:22.710 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:22.743 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:22.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 15 60827 60812 135.722 96.6875 0.00564606 0.00736294 2024-08-17T17:45:22.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:22.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:22 smithi175 ceph-mon[101434]: pgmap v1353: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-08-17T17:45:22.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:22 smithi064 ceph-mon[104516]: pgmap v1353: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-08-17T17:45:22.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:22 smithi064 ceph-mon[109246]: pgmap v1353: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-08-17T17:45:22.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:23.021 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:23.123 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:23.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:23.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:23.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:23.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:23.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:23.410 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:45:23.420 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:23.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:23.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.596 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:23.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:23.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:23.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:23.804 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 62345 62329 134.311 94.8125 0.0143591 0.00743994 2024-08-17T17:45:23.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:23.864 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:23.931 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:23.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:24.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:24.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:45:24.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:24.105 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:45:24.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:24.137 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:24.142 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:45:24.143 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:24.161 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:45:24.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:24.178 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:45:24.178 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:23] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-08-17T17:45:24.238 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:24.260 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:24.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:24.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:24.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:24.511 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:24.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:24.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:24.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:24.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:45:24.662 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:24.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:45:24.701 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:24.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:45:24.735 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:24.744 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:24.768 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:45:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:24.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:24 smithi064 ceph-mon[104516]: pgmap v1354: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:45:24.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4281320085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:24.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:24 smithi064 ceph-mon[109246]: pgmap v1354: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:45:24.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4281320085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:24.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:24.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 63072 63056 131.349 45.4375 0.00646031 0.00760837 2024-08-17T17:45:24.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:24.883 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:24 smithi175 ceph-mon[101434]: pgmap v1354: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:45:24.883 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4281320085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:24.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:24.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:25.020 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:25.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:25.168 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:25.209 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:25.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:25.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:25.444 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:25.480 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:25.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:25.582 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:25.619 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:25.721 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.761 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:25.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:25.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 64546 64530 130.083 92.125 0.0112127 0.00768285 2024-08-17T17:45:25.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:25.894 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:25.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:25.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:25.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:26.014 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.045 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:26.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:26.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:26.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:26.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:26.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:26.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:26.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.478 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:26.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:26.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:26.581 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:26 smithi175 ceph-mon[101434]: pgmap v1355: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:45:26.581 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]} : dispatch 2024-08-17T17:45:26.581 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-08-17T17:45:26.609 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:26.618 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:65633 10.c e423/410 10:3ab6062c:::benchmark_data_smithi064_306376_object65632:head v 423'4179, mlcod=423'4177)", 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.606719+0000", 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.010332648, 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010361446, 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606719+0000", 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606719+0000", 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606718+0000", 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606741+0000", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2787999999999999e-05 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606756+0000", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4440999999999999e-05 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606762+0000", 2024-08-17T17:45:26.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606899+0000", 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013632100000000001 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.606917+0000", 2024-08-17T17:45:26.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8861000000000001e-05 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:65644 10.1 10:85053722:::benchmark_data_smithi064_306376_object65643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.608654+0000", 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0083978679999999993, 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0085057250000000004, 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:26.623 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 65644 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608654+0000", 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608654+0000", 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608653+0000", 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608690+0000", 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7001e-05 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608700+0000", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8989999999999996e-06 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608705+0000", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5160000000000002e-06 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608750+0000", 2024-08-17T17:45:26.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4801e-05 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608908+0000", 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015762600000000001 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609043+0000", 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013554699999999999 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:26.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.613853+0000", 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0048096880000000003 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:65645 10.1 10:8d62ee12:::benchmark_data_smithi064_306376_object65644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.608981+0000", 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0080705770000000007, 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.008266453, 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:26.628 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 65645 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608981+0000", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608981+0000", 2024-08-17T17:45:26.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.608980+0000", 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609017+0000", 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7203999999999998e-05 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609040+0000", 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2745999999999999e-05 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609048+0000", 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0290000000000005e-06 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.611312+0000", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002263953 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.611431+0000", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011883900000000001 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.611552+0000", 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121644 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.613858+0000", 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002305709 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:65650 10.d 10:b4b0fb99:::benchmark_data_smithi064_306376_object65649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.609467+0000", 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0075849139999999999, 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0078667879999999996, 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 65650 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.634 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609467+0000", 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609467+0000", 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609466+0000", 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609512+0000", 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5954000000000001e-05 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609526+0000", 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.42e-05 2024-08-17T17:45:26.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609533+0000", 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3549999999999999e-06 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609577+0000", 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3433999999999998e-05 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609713+0000", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013611799999999999 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609857+0000", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000144318 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.612572+0000", 2024-08-17T17:45:26.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0027144140000000001 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.613099+0000", 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00052770499999999995 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:65651 10.1 10:8b884e74:::benchmark_data_smithi064_306376_object65650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-08-17T17:45:26.639 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.609918+0000", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0071333719999999998, 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0074938690000000002, 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 65651 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609918+0000", 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609918+0000", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609917+0000", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609971+0000", 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3106999999999998e-05 2024-08-17T17:45:26.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609982+0000", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0942e-05 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.609989+0000", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0360000000000001e-06 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.613880+0000", 2024-08-17T17:45:26.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038913590000000001 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614008+0000", 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000128245 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614248+0000", 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023962699999999999 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:65646 10.6 e423/410 10:682d1c9c:::benchmark_data_smithi064_306376_object65645:head v 423'4060, mlcod=422'4058)", 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:26.614110+0000", 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0029421680000000002, 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033700499999999999, 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614110+0000", 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614110+0000", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614109+0000", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614135+0000", 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6183999999999999e-05 2024-08-17T17:45:26.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614146+0000", 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0973e-05 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614153+0000", 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:26.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614378+0000", 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000225314 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:26.614399+0000", 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0687999999999999e-05 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:45:26.647 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 6 2024-08-17T17:45:26.648 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:45:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[104516]: pgmap v1355: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:45:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]} : dispatch 2024-08-17T17:45:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-08-17T17:45:26.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[109246]: pgmap v1355: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:45:26.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]} : dispatch 2024-08-17T17:45:26.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-08-17T17:45:26.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:26.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:26.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 65832 65816 128.53 80.375 0.0184756 0.00777424 2024-08-17T17:45:26.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:26.878 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:26.980 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:27.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:27.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:27.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:27.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:27.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:27.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:27.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:27.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:27.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.442 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:27.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:27.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:27.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:45:27.579 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:27.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:27.728 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:27.729 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:27 smithi175 ceph-mon[101434]: osdmap e423: 8 total, 8 up, 7 in 2024-08-17T17:45:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:27.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 67056 67040 126.953 76.5 0.00415026 0.00787126 2024-08-17T17:45:27.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:27.862 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:27.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:27 smithi064 ceph-mon[104516]: osdmap e423: 8 total, 8 up, 7 in 2024-08-17T17:45:27.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:27 smithi064 ceph-mon[109246]: osdmap e423: 8 total, 8 up, 7 in 2024-08-17T17:45:27.945 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:27.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:27.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:28.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:45:28.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:28.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:28.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:28.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:28.267 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:28.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:28.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:28.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:28.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:28.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:28 smithi175 ceph-mon[101434]: pgmap v1357: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:45:28.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:28 smithi175 ceph-mon[101434]: osdmap e424: 8 total, 8 up, 7 in 2024-08-17T17:45:28.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:28.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:28.794 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:28.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:28 smithi064 ceph-mon[104516]: pgmap v1357: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:45:28.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:28 smithi064 ceph-mon[104516]: osdmap e424: 8 total, 8 up, 7 in 2024-08-17T17:45:28.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:28 smithi064 ceph-mon[109246]: pgmap v1357: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:45:28.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:28 smithi064 ceph-mon[109246]: osdmap e424: 8 total, 8 up, 7 in 2024-08-17T17:45:28.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 68172 68156 125.27 69.75 0.0107582 0.00797703 2024-08-17T17:45:28.836 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:28.938 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:28.991 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:29.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:29.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:29.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:29.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:29.359 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:29.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:29.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:29.508 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:29.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:29.595 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:45:29.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1175962974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1175962974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:29.689 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:29.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:29.791 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:29.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 69462 69446 123.995 80.625 0.00779728 0.00805909 2024-08-17T17:45:29.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:29.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:29.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1175962974' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:45:29.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:29.970 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:30.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:30.106 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:30.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:30.240 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:30.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:30.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:30.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:30.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:30.592 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:45:30.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:30.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:30 smithi175 ceph-mon[101434]: pgmap v1359: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:45:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:45:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:30 smithi175 ceph-mon[101434]: osdmap e425: 8 total, 8 up, 7 in 2024-08-17T17:45:30.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:30.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:30.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:30.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:30.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 70734 70718 122.759 79.5 0.013594 0.00814025 2024-08-17T17:45:30.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[104516]: pgmap v1359: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:45:30.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:45:30.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[104516]: osdmap e425: 8 total, 8 up, 7 in 2024-08-17T17:45:30.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:30.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[109246]: pgmap v1359: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:45:30.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:45:30.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[109246]: osdmap e425: 8 total, 8 up, 7 in 2024-08-17T17:45:30.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:30.865 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:45:30.865 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:45:30.865 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:45:30.865 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:45:30.865 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:45:30.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:30.919 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:31.021 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:31.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:31.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:31.191 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:31.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:31.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:31.331 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:31.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:31.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:31.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:31.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:45:31.610 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:31.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:31 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:31 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:31.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:31 smithi175 ceph-mon[101434]: osdmap e426: 8 total, 8 up, 7 in 2024-08-17T17:45:31.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:31.744 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:31.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 72058 72042 121.677 82.75 0.00902453 0.00821266 2024-08-17T17:45:31.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:31.876 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[104516]: osdmap e426: 8 total, 8 up, 7 in 2024-08-17T17:45:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1613067894' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:31 smithi064 ceph-mon[109246]: osdmap e426: 8 total, 8 up, 7 in 2024-08-17T17:45:31.943 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:31.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:32.007 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:32.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:32.142 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:32.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.267 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:32.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:32.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:32.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:32.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:32.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:32.651 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:32.690 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:32.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:45:32.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:32.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 73221 73205 120.388 72.6875 0.00872564 0.00829846 2024-08-17T17:45:32.847 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:32.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:32 smithi064 ceph-mon[104516]: pgmap v1362: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 987 op/s 2024-08-17T17:45:32.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:32 smithi064 ceph-mon[109246]: pgmap v1362: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 987 op/s 2024-08-17T17:45:32.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:32 smithi175 ceph-mon[101434]: pgmap v1362: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 987 op/s 2024-08-17T17:45:32.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:32.989 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:33.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:33.113 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:33.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.264 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:33.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:33.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:33.397 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:33.449 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:45:33.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:33.485 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:45:33.485 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:33.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:33.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:45:33.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:33.541 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:33.550 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:45:33.550 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:33.585 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:45:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:33.620 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:45:33.620 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:33.642 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:33.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:45:33.656 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:45:33.656 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:33.675 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:33.697 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:45:33.697 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:33.716 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/761708892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:33.716 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/761708892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:33.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:33.805 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 74357 74341 119.121 71 0.0105336 0.00838965 2024-08-17T17:45:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.819 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:33.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:33.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/761708892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:33.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:33.945 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:34.047 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:34.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:34.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:33] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-08-17T17:45:34.223 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:34.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:34.373 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:34.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:34.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:34.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:34.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:34.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:34.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:34.764 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:34.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:34.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:45:34.806145+0000 min lat: 0.00139059 max lat: 0.515984 avg lat: 0.00845939 2024-08-17T17:45:34.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:45:34.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 75526 75510 117.969 73.0625 0.0027306 0.00845939 2024-08-17T17:45:34.817 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:34 smithi175 ceph-mon[101434]: pgmap v1363: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:45:34.836 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:34.836 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:34.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:34.888 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:34.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:34.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:34 smithi064 ceph-mon[104516]: pgmap v1363: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:45:34.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:34 smithi064 ceph-mon[109246]: pgmap v1363: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:45:34.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:35.022 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:35.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:35.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:35.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:35.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:35.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:35.402 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:35.504 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.544 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:35.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:35.645 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:35.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:35.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:35.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:35.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 76136 76120 116.022 38.125 0.00288464 0.00860056 2024-08-17T17:45:35.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:35.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:36.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:36.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:36.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:36.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:36.171 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:36.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:36.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:36.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:36.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:36.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:36.570 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:36.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:36.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:36.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:36.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 77062 77046 114.637 57.875 0.00713447 0.00871786 2024-08-17T17:45:36.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:36.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:36.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:36 smithi175 ceph-mon[101434]: pgmap v1364: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:45:36.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:36 smithi064 ceph-mon[104516]: pgmap v1364: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:45:36.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:36 smithi064 ceph-mon[109246]: pgmap v1364: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:45:36.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:36.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:36.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:37.084 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.132 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:37.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:37.268 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:37.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:37.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.411 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:37.512 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:37.545 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:37.598 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:45:37.646 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:37.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:37.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:37.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 78316 78300 113.794 78.375 0.0109574 0.00878257 2024-08-17T17:45:37.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:37.932 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:37.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:37.993 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:38.090 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:38.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:38.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:38.236 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:38.274 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:38.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:38.376 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:38.408 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:38.499 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:38.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:38.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:38.657 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:38.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:38 smithi064 ceph-mon[104516]: pgmap v1365: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:45:38.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:38 smithi064 ceph-mon[109246]: pgmap v1365: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:45:38.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:38.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:38.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 79559 79543 112.973 77.6875 0.0104007 0.00884677 2024-08-17T17:45:38.831 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:38 smithi175 ceph-mon[101434]: pgmap v1365: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:45:38.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:38.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:39.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:39.116 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:39.218 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:39.253 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:39.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:39.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:39.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:39.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:39.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:39.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:39.752 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:39.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:39.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 80714 80698 112.066 72.1875 0.00846411 0.00891786 2024-08-17T17:45:39.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:39.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:40.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:40.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:40.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:40.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:40.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:40.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:40.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:40.475 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:40.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:40.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:40.685 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:40 smithi175 ceph-mon[101434]: pgmap v1366: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:45:40.685 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:40.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:40.736 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:40.736 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:40.758 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:40.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 81917 81901 111.265 75.1875 0.00774804 0.00898242 2024-08-17T17:45:40.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:40 smithi064 ceph-mon[104516]: pgmap v1366: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:45:40.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:40.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:40 smithi064 ceph-mon[109246]: pgmap v1366: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:45:40.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:40.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:40.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:40.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:45:41.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:41.039 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:41.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:41.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:41.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:41.327 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:41.429 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:41.446 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:45:41.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:41.535 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:41.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:41.616 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:41.650 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:45:41.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:41.742 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:41.796 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:41.806 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 82997 82981 110.333 67.5 0.0102223 0.00905824 2024-08-17T17:45:41.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:41.875 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:41.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:41.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1215390613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:41 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1215390613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:41 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1215390613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:41 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:45:41.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:42.010 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:42.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:42.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:42.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:42.286 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:42.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:42.418 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:42.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:42.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:42.652 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:45:42.652 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:42.689 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:42.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:42.807 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 84246 84230 109.659 78.0625 0.0147408 0.00911449 2024-08-17T17:45:42.828 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:42.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:42.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[104516]: pgmap v1367: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:45:42.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:45:42.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[104516]: osdmap e427: 8 total, 8 up, 7 in 2024-08-17T17:45:42.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1146761802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[109246]: pgmap v1367: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:45:42.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:45:42.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[109246]: osdmap e427: 8 total, 8 up, 7 in 2024-08-17T17:45:42.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1146761802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:42 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:42 smithi175 ceph-mon[101434]: pgmap v1367: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:45:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:42 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:45:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:42 smithi175 ceph-mon[101434]: osdmap e427: 8 total, 8 up, 7 in 2024-08-17T17:45:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1146761802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:42 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:42.930 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:42.967 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:43.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:43.105 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:43.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:43.207 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:43.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:43.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:43.373 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:43.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:43.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:43.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:43.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:43.673 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:43.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:43.709 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:43 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:43.709 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:43 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:45:43.709 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:43 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:43.709 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:43 smithi175 ceph-mon[101434]: osdmap e428: 8 total, 8 up, 7 in 2024-08-17T17:45:43.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:45:43.762 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:43.762 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:43.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:43.807 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 85388 85372 108.877 71.375 0.0159687 0.00917905 2024-08-17T17:45:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.818 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:43.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:43.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:43.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:43.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:45:43.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:43.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[104516]: osdmap e428: 8 total, 8 up, 7 in 2024-08-17T17:45:43.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:43.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:45:43.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:43.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-mon[109246]: osdmap e428: 8 total, 8 up, 7 in 2024-08-17T17:45:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.961 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:43.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:44.062 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:44.101 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:44.203 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:44.246 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:44.257 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:43] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-08-17T17:45:44.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:44.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:44.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:44.502 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:44.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:44.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:44.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:44.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:44.770 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:44.807 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 86483 86467 108.069 68.4375 0.0122974 0.00924798 2024-08-17T17:45:44.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.913 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:44.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:44.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:44 smithi064 ceph-mon[104516]: pgmap v1370: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:45:44.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:44 smithi064 ceph-mon[109246]: pgmap v1370: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:45:45.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:45.050 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:45.152 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:45.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:44 smithi175 ceph-mon[101434]: pgmap v1370: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:45:45.188 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:45.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:45.325 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:45.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:45.454 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:45.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.596 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:45.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:45.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:45.738 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:45.798 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:45.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 87652 87636 107.381 73.0625 0.0255972 0.0093072 2024-08-17T17:45:45.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:45.866 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:45:45.866 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:45.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:45.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:45.900 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:45:45.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:45.931 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:45:45.931 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:45.962 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:45:45.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:45.985 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:45.994 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:45:45.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:46.023 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:46.032 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:45:46.032 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:46.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:45:46.065 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:46.096 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:45:46.096 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:46.124 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:46.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:46.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:46.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:46.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:46.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:46.433 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:46.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:46.560 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:46.662 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.706 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:46.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:46.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:45:46.806 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:46.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 88694 88678 106.568 65.125 0.0317756 0.0093788 2024-08-17T17:45:46.846 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:46.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[109246]: pgmap v1371: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T17:45:46.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1693645036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:46.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[104516]: pgmap v1371: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T17:45:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1693645036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:46.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:46 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:46.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:46.974 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:47.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:47.101 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:46 smithi175 ceph-mon[101434]: pgmap v1371: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T17:45:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1693645036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:47.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:46 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:47.203 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:47.212 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:47.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:45:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:47.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:47.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:47.387 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:47.489 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:47.510 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:47.612 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:47.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:47.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:47.776 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:47.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 89437 89421 105.434 46.4375 0.00518125 0.0094795 2024-08-17T17:45:47.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:47.912 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:48.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:48.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:48.148 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:48.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:48.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:48.324 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:48.426 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:48.468 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:48.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:48.570 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:48.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:48 smithi064 ceph-mon[104516]: pgmap v1372: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:45:48.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:48 smithi064 ceph-mon[109246]: pgmap v1372: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:45:48.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:48.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:48.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 90592 90576 104.818 72.1875 0.0141544 0.00953535 2024-08-17T17:45:48.841 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:48.874 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:48.926 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:48 smithi175 ceph-mon[101434]: pgmap v1372: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:45:48.976 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:49.061 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:49.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:49.207 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:49.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:49.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:49.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:49.459 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:49.472 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:49.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:49.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:49.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:49.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:49.763 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:49.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 91594 91578 104.051 62.625 0.0818931 0.00960413 2024-08-17T17:45:49.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:49.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:50.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:50.037 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:50.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:50.179 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:50.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:50.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:50.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:50.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:50.521 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:45:50.521 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:45:50.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:45:50.521 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:169821 8.10 8.c02c9950 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e292)", 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:31.898645+0000", 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 497.92132042399999, 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0042402159999999, 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 169821 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898645+0000", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898645+0000", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898644+0000", 2024-08-17T17:45:50.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898651+0000", 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7089999999999996e-06 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898661+0000", 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9850000000000001e-06 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898669+0000", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2930000000000001e-06 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898710+0000", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1365000000000001e-05 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.898718+0000", 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0469999999999994e-06 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902875+0000", 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0041566 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902886+0000", 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0702e-05 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.526 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:169886 8.10 8.fd3010 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e292)", 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:31.913063+0000", 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 497.90690332600002, 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98985983700000002, 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 169886 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913063+0000", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913063+0000", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913061+0000", 2024-08-17T17:45:50.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913067+0000", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7200000000000003e-06 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913080+0000", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2644e-05 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913086+0000", 2024-08-17T17:45:50.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2809999999999997e-06 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913135+0000", 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8834e-05 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.913143+0000", 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1729999999999995e-06 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902915+0000", 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98977201999999997 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902922+0000", 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5190000000000003e-06 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:169899 8.10 8.eefffe70 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e292)", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:31.922854+0000", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 497.89711197700001, 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.98009459200000004, 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 169899 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.532 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922854+0000", 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922854+0000", 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922852+0000", 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922859+0000", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1349999999999997e-06 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922875+0000", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6129000000000001e-05 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922882+0000", 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7340000000000008e-06 2024-08-17T17:45:50.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922945+0000", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2562000000000002e-05 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.922952+0000", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8719999999999996e-06 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902941+0000", 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.97998910299999997 2024-08-17T17:45:50.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902949+0000", 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5020000000000003e-06 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:169953 8.10 8.51bedf70 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e292)", 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:31.968215+0000", 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 497.85175067500001, 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93475776799999999, 2024-08-17T17:45:50.536 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 169953 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968215+0000", 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968215+0000", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968214+0000", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968220+0000", 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8279999999999999e-06 2024-08-17T17:45:50.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968231+0000", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.172e-05 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968239+0000", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4580000000000004e-06 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968283+0000", 2024-08-17T17:45:50.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3823999999999999e-05 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:31.968291+0000", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3049999999999993e-06 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902966+0000", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.93467508600000004 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:32.902973+0000", 2024-08-17T17:45:50.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8940000000000004e-06 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197593 8.15 8:aaadc3b1:::benchmark_data_smithi064_242541_object75100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.404127+0000", 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.41583867399999, 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.030282412, 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197593 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404127+0000", 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404127+0000", 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404126+0000", 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404132+0000", 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7849999999999997e-06 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404142+0000", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0427e-05 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404149+0000", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404216+0000", 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6999e-05 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.404226+0000", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0387999999999999e-05 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359734+0000", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.95550790799999996 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359745+0000", 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1409e-05 2024-08-17T17:45:50.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.429634+0000", 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0698883610000001 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.429741+0000", 2024-08-17T17:45:50.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107462 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.430130+0000", 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000388902 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434013+0000", 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003882927 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434027+0000", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3831e-05 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434063+0000", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5874999999999998e-05 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434279+0000", 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00021660399999999999 2024-08-17T17:45:50.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434313+0000", 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3226999999999999e-05 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434410+0000", 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6968999999999999e-05 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197636 8.15 8:aad276b8:::benchmark_data_smithi064_242541_object75143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.438156+0000", 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.38180980499999, 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0013621530000001, 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197636 2024-08-17T17:45:50.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438156+0000", 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438156+0000", 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438154+0000", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438162+0000", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2520000000000002e-06 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438175+0000", 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3115999999999999e-05 2024-08-17T17:45:50.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438182+0000", 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2339999999999997e-06 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438653+0000", 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00047133 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.438665+0000", 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2041999999999999e-05 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359783+0000", 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.92111793799999997 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359791+0000", 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1040000000000007e-06 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.432327+0000", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0725359880000001 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.432444+0000", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011646699999999999 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.432723+0000", 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027890599999999999 2024-08-17T17:45:50.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439147+0000", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0064242860000000004 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439201+0000", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3587e-05 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439248+0000", 2024-08-17T17:45:50.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7416000000000001e-05 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439375+0000", 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012712800000000001 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439417+0000", 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1761999999999998e-05 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439518+0000", 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000101269 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197644 8.15 8:ab1ae180:::benchmark_data_smithi064_242541_object75151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.442355+0000", 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.37761073000001, 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.999443018, 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.558 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197644 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442355+0000", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442355+0000", 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442354+0000", 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442362+0000", 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5469999999999997e-06 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442374+0000", 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2043e-05 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442381+0000", 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7159999999999997e-06 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442437+0000", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5653999999999999e-05 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.442446+0000", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4320000000000005e-06 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359812+0000", 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.91736584300000001 2024-08-17T17:45:50.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359820+0000", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6159999999999996e-06 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434451+0000", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0746313430000001 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434572+0000", 2024-08-17T17:45:50.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000120783 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.434831+0000", 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025933400000000001 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439258+0000", 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0044272840000000001 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439298+0000", 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9155e-05 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441574+0000", 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022766729999999999 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441627+0000", 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2568999999999998e-05 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441677+0000", 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0256999999999997e-05 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441798+0000", 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121171 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.567 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197660 8.15 8:a940eba8:::benchmark_data_smithi064_242541_object75167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.453277+0000", 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.36668854200002, 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9889091749999999, 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.568 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197660 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453277+0000", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453277+0000", 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453276+0000", 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453283+0000", 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8329999999999996e-06 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453296+0000", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3076e-05 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453303+0000", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6369999999999999e-06 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453363+0000", 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9857e-05 2024-08-17T17:45:50.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.453374+0000", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0750999999999999e-05 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359846+0000", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.90647231399999995 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359854+0000", 2024-08-17T17:45:50.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1040000000000007e-06 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.436683+0000", 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0768286520000001 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.436804+0000", 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000121494 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.437196+0000", 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00039113899999999998 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439264+0000", 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0020684570000000001 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441863+0000", 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025988589999999998 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441916+0000", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3579999999999999e-05 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442025+0000", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000108595 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442072+0000", 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7373000000000002e-05 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442187+0000", 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114129 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197698 8.15 8:af19217a:::benchmark_data_smithi064_242541_object75205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.483037+0000", 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.33692903500003, 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9613281470000001, 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.576 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197698 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483037+0000", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483037+0000", 2024-08-17T17:45:50.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483035+0000", 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483043+0000", 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8550000000000002e-06 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483060+0000", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6833999999999998e-05 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483069+0000", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7860000000000002e-06 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483158+0000", 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.9371999999999997e-05 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.483170+0000", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1697e-05 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359876+0000", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.87670672900000002 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359884+0000", 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8210000000000002e-06 2024-08-17T17:45:50.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439567+0000", 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0796826129999999 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.439697+0000", 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012975000000000001 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.440049+0000", 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00035289099999999998 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.441929+0000", 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001879976 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442228+0000", 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000298208 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442271+0000", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2930999999999999e-05 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444202+0000", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0019313589999999999 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444250+0000", 2024-08-17T17:45:50.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8519000000000001e-05 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444365+0000", 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114591 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197721 8.15 8:aa536397:::benchmark_data_smithi064_242541_object75228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.498895+0000", 2024-08-17T17:45:50.584 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.32107120900002, 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9524110559999999, 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197721 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498895+0000", 2024-08-17T17:45:50.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498895+0000", 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498893+0000", 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498900+0000", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6059999999999999e-06 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498915+0000", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5387e-05 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498921+0000", 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1029999999999998e-06 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498967+0000", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5592e-05 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.498974+0000", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1910000000000003e-06 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359960+0000", 2024-08-17T17:45:50.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.86098611599999997 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359969+0000", 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8680000000000005e-06 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442296+0000", 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0823267379999999 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442415+0000", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000118595 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.442721+0000", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00030659600000000002 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.450912+0000", 2024-08-17T17:45:50.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0081903670000000005 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.450970+0000", 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8539999999999999e-05 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451017+0000", 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7275999999999999e-05 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451111+0000", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3871999999999998e-05 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451194+0000", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.2755000000000003e-05 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451306+0000", 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011179999999999999 2024-08-17T17:45:50.592 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197757 8.15 8:a9570d97:::benchmark_data_smithi064_242541_object75264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.532749+0000", 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.28721717600001, 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9211763390000001, 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.593 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197757 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532749+0000", 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532749+0000", 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532748+0000", 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532755+0000", 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7970000000000001e-06 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532768+0000", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2255e-05 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.532774+0000", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5889999999999999e-06 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.533514+0000", 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00073956399999999995 2024-08-17T17:45:50.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.533524+0000", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0251000000000001e-05 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.359998+0000", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.82647422699999995 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360006+0000", 2024-08-17T17:45:50.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6599999999999995e-06 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444411+0000", 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.084405142 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444489+0000", 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8332999999999997e-05 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.444744+0000", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025495999999999998 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451054+0000", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.006310141 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451090+0000", 2024-08-17T17:45:50.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5574999999999997e-05 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453730+0000", 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026398149999999998 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453779+0000", 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8789000000000003e-05 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453811+0000", 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2208000000000001e-05 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453925+0000", 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011424799999999999 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.601 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197796 8.15 8:aee50e6b:::benchmark_data_smithi064_242541_object75303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.567088+0000", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.252877931, 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.890393032, 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197796 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567088+0000", 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567088+0000", 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567086+0000", 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567094+0000", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5889999999999997e-06 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567108+0000", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4253999999999999e-05 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567115+0000", 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1269999999999997e-06 2024-08-17T17:45:50.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567172+0000", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6382999999999999e-05 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567182+0000", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0079e-05 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360028+0000", 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.79284608899999998 2024-08-17T17:45:50.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360035+0000", 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3289999999999996e-06 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.446668+0000", 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0866330120000001 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.446772+0000", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104232 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.447135+0000", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036271600000000001 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453737+0000", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0066015969999999998 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453976+0000", 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023923799999999999 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.454723+0000", 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00074693199999999996 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457320+0000", 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025972920000000002 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457373+0000", 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3062999999999997e-05 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457481+0000", 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107842 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197797 8.15 8:a8f26c3e:::benchmark_data_smithi064_242541_object75304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.609 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.567851+0000", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.25211463599999, 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8978638370000001, 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197797 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567851+0000", 2024-08-17T17:45:50.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567851+0000", 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567850+0000", 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567858+0000", 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4900000000000003e-06 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567870+0000", 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2296000000000001e-05 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567877+0000", 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2919999999999998e-06 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567923+0000", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5787000000000001e-05 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.567932+0000", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7139999999999999e-06 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360127+0000", 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.792195435 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360136+0000", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7539999999999995e-06 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.448735+0000", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.088598894 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.448880+0000", 2024-08-17T17:45:50.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014516799999999999 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.449178+0000", 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00029797999999999999 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.453744+0000", 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0045657290000000001 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.454687+0000", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00094340699999999995 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460060+0000", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0053731170000000002 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465552+0000", 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0054915060000000002 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465597+0000", 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5395999999999997e-05 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465715+0000", 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000117769 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197827 8.15 8:ad5650ed:::benchmark_data_smithi064_242541_object75334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.617 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.591894+0000", 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.22807162800001, 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8796253620000001, 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197827 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591894+0000", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591894+0000", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591893+0000", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591899+0000", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5740000000000001e-06 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591922+0000", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2368999999999999e-05 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.591929+0000", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4200000000000001e-06 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.592009+0000", 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9336000000000001e-05 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.592019+0000", 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0893999999999999e-05 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360170+0000", 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.76815097099999996 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360179+0000", 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1729999999999995e-06 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451333+0000", 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0911545659999999 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451447+0000", 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000113583 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.451797+0000", 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00035059799999999998 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457521+0000", 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0057239489999999999 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.459986+0000", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0024642710000000001 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460095+0000", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000109755 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471368+0000", 2024-08-17T17:45:50.624 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011273050999999999 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471417+0000", 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.8436999999999997e-05 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471520+0000", 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010281999999999999 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.625 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197840 8.15 8:a8331242:::benchmark_data_smithi064_242541_object75347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.606397+0000", 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.21356933300001, 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.865360372, 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197840 2024-08-17T17:45:50.626 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606397+0000", 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606397+0000", 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.627 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606395+0000", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606401+0000", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8240000000000002e-06 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606417+0000", 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5199e-05 2024-08-17T17:45:50.628 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606424+0000", 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0099999999999998e-06 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606471+0000", 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7343999999999999e-05 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.629 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.606482+0000", 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0722e-05 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360217+0000", 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.75373510200000005 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360223+0000", 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.2179999999999996e-06 2024-08-17T17:45:50.630 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.454747+0000", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0945241050000001 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.454888+0000", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000141125 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.455326+0000", 2024-08-17T17:45:50.631 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00043747300000000002 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.459943+0000", 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0046171129999999999 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460026+0000", 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3504000000000006e-05 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.632 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460131+0000", 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104934 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471623+0000", 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011491285 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471665+0000", 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2258000000000001e-05 2024-08-17T17:45:50.633 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471757+0000", 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2064999999999994e-05 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197864 8.15 8:ad7108f6:::benchmark_data_smithi064_242541_object75371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.628641+0000", 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.19132494799999, 2024-08-17T17:45:50.634 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8433307889999999, 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197864 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628641+0000", 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.635 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628641+0000", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628639+0000", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628650+0000", 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0721e-05 2024-08-17T17:45:50.636 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628667+0000", 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7059999999999999e-05 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628675+0000", 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1030000000000002e-06 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.637 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628731+0000", 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6082999999999998e-05 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.628742+0000", 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0712e-05 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360251+0000", 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.73150902600000001 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.638 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360256+0000", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0030000000000002e-06 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457543+0000", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0972874479999999 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.457650+0000", 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000106697 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.639 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.458011+0000", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036135799999999999 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465416+0000", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0074045079999999997 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465455+0000", 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9539000000000002e-05 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.640 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471593+0000", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0061381420000000001 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471836+0000", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00024238100000000001 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471878+0000", 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2279e-05 2024-08-17T17:45:50.641 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471972+0000", 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3845999999999996e-05 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197866 8.15 8:a95192f8:::benchmark_data_smithi064_242541_object75373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.630252+0000", 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.18971387599998, 2024-08-17T17:45:50.642 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.84192825, 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197866 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630252+0000", 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.643 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630252+0000", 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630251+0000", 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.644 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630256+0000", 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5639999999999998e-06 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630266+0000", 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0421000000000001e-05 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630271+0000", 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1930000000000001e-06 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.645 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630304+0000", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3049999999999997e-05 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.630311+0000", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2010000000000002e-06 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360277+0000", 2024-08-17T17:45:50.646 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.72996561100000001 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360283+0000", 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.733e-06 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460171+0000", 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0998882510000001 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.647 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460281+0000", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.460690+0000", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00040889000000000002 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465460+0000", 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0047703210000000001 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.648 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471562+0000", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0061020720000000001 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471793+0000", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023067199999999999 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472045+0000", 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025199599999999999 2024-08-17T17:45:50.649 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472086+0000", 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1455000000000002e-05 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472180+0000", 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3789999999999995e-05 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.650 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197869 8.15 8:a9eb9510:::benchmark_data_smithi064_242541_object75376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.632880+0000", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.18708626699998, 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.853895673, 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197869 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.651 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632880+0000", 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632880+0000", 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632879+0000", 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632883+0000", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1189999999999999e-06 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632897+0000", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4384e-05 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632902+0000", 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0239999999999997e-06 2024-08-17T17:45:50.653 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632952+0000", 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9094999999999997e-05 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.632962+0000", 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0355e-05 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.654 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360318+0000", 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.72735626600000003 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360328+0000", 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6299999999999993e-06 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.462770+0000", 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.102442632 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.655 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.462912+0000", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014178800000000001 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.463287+0000", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000374414 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471331+0000", 2024-08-17T17:45:50.656 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0080443839999999999 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472007+0000", 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00067585099999999999 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472214+0000", 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000206765 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.657 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486612+0000", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.01439792 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486664+0000", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2348000000000001e-05 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486775+0000", 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000111406 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197930 8.15 8:ac94f3e5:::benchmark_data_smithi064_242541_object75437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.745786+0000", 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 462.074180394, 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7411462310000001, 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.659 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197930 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745786+0000", 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745786+0000", 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.660 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745784+0000", 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745790+0000", 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3230000000000003e-06 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745804+0000", 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3352e-05 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.661 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745810+0000", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5780000000000004e-06 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745881+0000", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0355e-05 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.745891+0000", 2024-08-17T17:45:50.662 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0142e-05 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360391+0000", 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.61450065899999995 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360399+0000", 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0129999999999993e-06 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.663 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465770+0000", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.105370236 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.465884+0000", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000114623 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.466290+0000", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00040578400000000002 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.664 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.471529+0000", 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0052392360000000004 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472250+0000", 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00072048399999999997 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.474783+0000", 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025331189999999999 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.665 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486816+0000", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.012033398000000001 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486872+0000", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5989000000000002e-05 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.486932+0000", 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9423999999999999e-05 2024-08-17T17:45:50.666 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:197968 8.15 8:a857527a:::benchmark_data_smithi064_242541_object75475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:08.162844+0000", 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 461.65712165100001, 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.324516561, 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 197968 2024-08-17T17:45:50.667 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162844+0000", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162844+0000", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162843+0000", 2024-08-17T17:45:50.668 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162850+0000", 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0929999999999997e-06 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162873+0000", 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2985000000000001e-05 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.669 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162880+0000", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4420000000000001e-06 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162924+0000", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4053e-05 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.162932+0000", 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8879999999999997e-06 2024-08-17T17:45:50.670 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360420+0000", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.197487838 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.360428+0000", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9039999999999992e-06 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.468753+0000", 2024-08-17T17:45:50.671 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.108325244 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.468862+0000", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010839400000000001 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4,6", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.469253+0000", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00039136599999999998 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.472529+0000", 2024-08-17T17:45:50.672 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032764539999999998 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.474793+0000", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0022636599999999998 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.474841+0000", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7679000000000002e-05 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.487219+0000", 2024-08-17T17:45:50.673 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.012377826999999999 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.487261+0000", 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2599000000000003e-05 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:09.487361+0000", 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9612999999999996e-05 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.674 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:50.675 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:50.675 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:45:50.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:50.701 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:45:50.733 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:50.809 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 92403 92387 103.096 50.5625 0.00578857 0.0096949 2024-08-17T17:45:50.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:50.866 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:50.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:45:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:50 smithi175 ceph-mon[101434]: pgmap v1373: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2024-08-17T17:45:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4160727765' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:50 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[104516]: pgmap v1373: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2024-08-17T17:45:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4160727765' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[109246]: pgmap v1373: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 62 MiB/s wr, 986 op/s 2024-08-17T17:45:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4160727765' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:50 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:45:50.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:51.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:51.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:51.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:51.264 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:51.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:51.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:51.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:45:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.448 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:51.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:51.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:51.583 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:51.686 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:51.718 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:51.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:45:51.809 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 93499 93483 102.488 68.5 0.00636434 0.00975189 2024-08-17T17:45:51.820 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:51.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:51.977 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:51 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:45:51.977 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:51 smithi175 ceph-mon[101434]: osdmap e429: 8 total, 8 up, 7 in 2024-08-17T17:45:52.015 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:52.069 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:52.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:51 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:45:52.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:51 smithi064 ceph-mon[104516]: osdmap e429: 8 total, 8 up, 7 in 2024-08-17T17:45:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:51 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:45:52.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:51 smithi064 ceph-mon[109246]: osdmap e429: 8 total, 8 up, 7 in 2024-08-17T17:45:52.219 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:52.223 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:52.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:52.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:52.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:52.534 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:45:52.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:52.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.688 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:52.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:52.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:52.809 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 94490 94474 101.788 61.9375 0.0233747 0.00981887 2024-08-17T17:45:52.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.835 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:52.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:52.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:52 smithi064 ceph-mon[109246]: pgmap v1375: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:45:52.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:52 smithi064 ceph-mon[104516]: pgmap v1375: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:45:52.972 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:52 smithi175 ceph-mon[101434]: pgmap v1375: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:45:52.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:53.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.115 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:53.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:53.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:53.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:53.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:53.384 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:53.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:53.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:53.526 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:53.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:53.662 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:53.742 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:45:53.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:53.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:53.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 95528 95512 101.162 64.875 0.00393362 0.00987588 2024-08-17T17:45:53.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:53.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:54.083 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:54.131 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:54.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:53 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/640905586' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:53 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:53 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/640905586' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:53 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:45:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:45:53] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-08-17T17:45:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:53 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/640905586' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:53 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:54.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:54.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:54.366 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:54.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:54.510 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.542 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:54.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:54.643 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:54.676 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:54.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:54.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:45:54.810282+0000 min lat: 0.00139059 max lat: 0.515984 avg lat: 0.00994115 2024-08-17T17:45:54.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:45:54.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 96547 96531 100.538 63.6875 0.00992048 0.00994115 2024-08-17T17:45:54.812 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:54.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[104516]: pgmap v1376: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:45:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:54.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[104516]: osdmap e430: 8 total, 8 up, 7 in 2024-08-17T17:45:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[109246]: pgmap v1376: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:45:54.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:54.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:54.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:54 smithi064 ceph-mon[109246]: osdmap e430: 8 total, 8 up, 7 in 2024-08-17T17:45:54.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:55.014 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:54 smithi175 ceph-mon[101434]: pgmap v1376: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:45:55.014 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:54 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:55.014 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:55.014 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:54 smithi175 ceph-mon[101434]: osdmap e430: 8 total, 8 up, 7 in 2024-08-17T17:45:55.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:55.080 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:55.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:55.212 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:55.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:55.342 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:55.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:55.474 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:55.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:55.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:55.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:55.740 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:55.764 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:45:55.764 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:97510 10.7 10:ea95ed59:::benchmark_data_smithi064_306376_object97509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:55.752724+0000", 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.010289207, 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010321954, 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 97510 2024-08-17T17:45:55.765 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752724+0000", 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752724+0000", 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:55.766 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752723+0000", 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752795+0000", 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1503999999999998e-05 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752806+0000", 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1044e-05 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.767 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.752811+0000", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.7389999999999996e-06 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.753533+0000", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00072186600000000004 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.753658+0000", 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000125084 2024-08-17T17:45:55.768 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.753792+0000", 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000133631 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761743+0000", 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079507520000000002 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.769 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:97515 10.9 10:99af2286:::benchmark_data_smithi064_306376_object97514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:55.756937+0000", 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.0060762569999999998, 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0061923330000000004, 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 97515 2024-08-17T17:45:55.770 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.756937+0000", 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.756937+0000", 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.771 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.756936+0000", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757017+0000", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0525999999999994e-05 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757034+0000", 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7370000000000001e-05 2024-08-17T17:45:55.772 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757041+0000", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6939999999999998e-06 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757111+0000", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0364000000000006e-05 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757245+0000", 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013387699999999999 2024-08-17T17:45:55.773 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757413+0000", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016816300000000001 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761228+0000", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0038152030000000001 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761362+0000", 2024-08-17T17:45:55.774 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013375999999999999 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:97516 10.7 10:ebbd37e9:::benchmark_data_smithi064_306376_object97515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:55.757975+0000", 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.005038654, 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0052321039999999996, 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:55.775 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 97516 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757975+0000", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:55.776 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757975+0000", 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.757974+0000", 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.758030+0000", 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5186000000000003e-05 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.777 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.758040+0000", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0937e-05 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.758047+0000", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3330000000000002e-06 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761767+0000", 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037204740000000001 2024-08-17T17:45:55.778 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761900+0000", 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000132996 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762036+0000", 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000135332 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.779 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:97512 10.a e430/410 10:571dc8e0:::benchmark_data_smithi064_306376_object97511:head v 430'6196, mlcod=430'6192)", 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:55.761905+0000", 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.001108522, 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00136651, 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761905+0000", 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.780 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761905+0000", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761905+0000", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761937+0000", 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2848000000000001e-05 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.781 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761948+0000", 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0328e-05 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.761954+0000", 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9680000000000003e-06 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.782 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762009+0000", 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4929000000000002e-05 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762027+0000", 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8345000000000001e-05 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.783 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:97521 10.3 10:cfa29751:::benchmark_data_smithi064_306376_object97520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:55.762640+0000", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 0.00037347900000000002, 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00068569799999999999, 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 97521 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762640+0000", 2024-08-17T17:45:55.784 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762640+0000", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762639+0000", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762680+0000", 2024-08-17T17:45:55.785 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1096000000000001e-05 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762693+0000", 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3518e-05 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762700+0000", 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3899999999999998e-06 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.786 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762738+0000", 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8704000000000001e-05 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762863+0000", 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012434999999999999 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:45:55.787 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:55.762979+0000", 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011595399999999999 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: ], 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 5 2024-08-17T17:45:55.788 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:45:55.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 15 97565 97550 99.9335 63.6875 0.00705999 0.0100015 2024-08-17T17:45:55.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:55.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:55.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:55.881 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:55.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:55.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:55.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:56.031 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:56.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:56.190 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:56.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:56.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.329 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:56.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:56.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:56.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:56.562 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:56.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:56.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:56.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:56.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 98639 98623 99.4032 67.0625 0.00869872 0.0100552 2024-08-17T17:45:56.831 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:56 smithi175 ceph-mon[101434]: pgmap v1378: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:45:56.837 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:56.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:56.976 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:45:57.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:56 smithi064 ceph-mon[104516]: pgmap v1378: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:45:57.105 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:56 smithi064 ceph-mon[109246]: pgmap v1378: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:45:57.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:57.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:57.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:45:57.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:57.297 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:57.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.610 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:57.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:57.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:57.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:57.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:57.810 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 99641 99625 98.8194 62.625 0.00898603 0.010114 2024-08-17T17:45:57.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:57.866 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:57.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:57.990 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:58.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:58.133 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.271 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:58.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:58.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:58.414 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:58.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:58.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:58.648 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:58.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:58.812 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 100434 100418 98.0463 49.5625 0.00991757 0.0101941 2024-08-17T17:45:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:58.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:58.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:58 smithi064 ceph-mon[104516]: pgmap v1379: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:45:58.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:58 smithi064 ceph-mon[109246]: pgmap v1379: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:45:58.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:58.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:58 smithi175 ceph-mon[101434]: pgmap v1379: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:45:59.081 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:59.105 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:59.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:59.242 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:59.344 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:59.376 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:59.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:59.499 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:59.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:59.529 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:45:59.529 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:59.534 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:59.547 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:45:59.547 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:59.572 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:45:59.572 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:59.596 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:45:59.596 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:59.601 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:59.614 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:45:59.614 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:59.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:59.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:59.665 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:45:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:59.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:45:59.707 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:59.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:59.733 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:45:59.733 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:59.752 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:45:59.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:45:59.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:59.812 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 101128 101112 97.2052 43.375 0.00528996 0.0102814 2024-08-17T17:45:59.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:59.897 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:45:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:59.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:00.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:00.056 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/503128744' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/625866325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/503128744' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:45:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/625866325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:00.128 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:00.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/503128744' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:59 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:46:00.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:45:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/625866325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.168 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:00.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:00.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:00.296 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:00.398 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:00.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:00.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.572 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:00.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:00.672 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:00.710 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:00.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:00.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 102033 102017 96.5894 56.5625 0.00969063 0.0103478 2024-08-17T17:46:00.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:00 smithi175 ceph-mon[101434]: pgmap v1380: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:00 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:46:00.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:00 smithi175 ceph-mon[101434]: osdmap e431: 8 total, 8 up, 7 in 2024-08-17T17:46:00.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:00.976 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:01.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:01.106 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[104516]: pgmap v1380: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:46:01.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[104516]: osdmap e431: 8 total, 8 up, 7 in 2024-08-17T17:46:01.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[109246]: pgmap v1380: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:46:01.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:00 smithi064 ceph-mon[109246]: osdmap e431: 8 total, 8 up, 7 in 2024-08-17T17:46:01.208 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:01.250 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:01.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:01.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:01.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:01.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:01.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:01.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:01.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:01.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:01.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 102892 102876 95.9491 53.6875 0.0241597 0.0104166 2024-08-17T17:46:01.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:01.914 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:02.016 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:02.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:02.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:02.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:02.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:02.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:02.436 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:02.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:46:02.486 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:02.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.638 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:02.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:02.739 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.780 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:02.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:02.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 103853 103837 95.4213 60.0625 0.0147563 0.0104744 2024-08-17T17:46:02.845 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:02.845 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:02.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:02.921 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:02.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:03.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:03.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:03.157 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:02 smithi175 ceph-mon[101434]: pgmap v1382: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:46:03.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:02 smithi064 ceph-mon[104516]: pgmap v1382: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:46:03.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:02 smithi064 ceph-mon[109246]: pgmap v1382: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2024-08-17T17:46:03.192 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:03.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.359 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:03.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:03.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:03.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:03.635 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:03.737 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:03.782 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:03.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 104833 104817 94.9261 61.25 0.0101697 0.0105296 2024-08-17T17:46:03.884 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:03.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:03.964 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:46:04.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:04.056 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:04.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:04.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:04.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:04.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:04.330 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:04.340 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:04 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:04 smithi175 ceph-mon[101434]: pgmap v1383: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-17T17:46:04.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[104516]: pgmap v1383: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-17T17:46:04.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:03] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-08-17T17:46:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:04.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:04 smithi064 ceph-mon[109246]: pgmap v1383: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-17T17:46:04.468 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:04.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:04.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:04.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:04.748 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:04.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 105792 105776 94.4262 59.9375 0.00817623 0.0105853 2024-08-17T17:46:04.850 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:04.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:04.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:04.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:05.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:05.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:05.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:05.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:05.203 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:05.204 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:46:05.285 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:05.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1265965943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:05 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1265965943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:05 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1265965943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:05 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:05.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:05.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:05.600 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:05.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:05.734 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:05.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 106805 106789 93.9879 63.3125 0.0178197 0.0106344 2024-08-17T17:46:05.836 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:05.884 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:05.986 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:06.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:06.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:06.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:46:06.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:06.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:06.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:06.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:06.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:06 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:06 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:06 smithi175 ceph-mon[101434]: osdmap e432: 8 total, 8 up, 7 in 2024-08-17T17:46:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:06 smithi175 ceph-mon[101434]: pgmap v1385: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s 2024-08-17T17:46:06.437 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[104516]: osdmap e432: 8 total, 8 up, 7 in 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[104516]: pgmap v1385: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[109246]: osdmap e432: 8 total, 8 up, 7 in 2024-08-17T17:46:06.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:06 smithi064 ceph-mon[109246]: pgmap v1385: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s 2024-08-17T17:46:06.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:06.580 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:06.596 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:06.597 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:06.681 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:06.719 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:06.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 107770 107754 93.5202 60.3125 0.0286624 0.0106877 2024-08-17T17:46:06.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:06.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:06.968 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:07.005 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:07.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:07.154 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:07.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:07.293 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:07.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:46:07.395 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.430 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:07.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:07.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:07.564 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:07.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:07.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:07.662 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:07.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:07.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:07.708 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:07.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:07.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 108758 108742 93.0849 61.75 0.0119406 0.0107378 2024-08-17T17:46:07.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:07.896 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:07.937 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:07.972 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:08.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:08.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:08.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:08.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:08.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:08.379 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:08.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:08.513 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:08.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:08.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:08.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:08.767 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:08 smithi064 ceph-mon[104516]: pgmap v1386: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:46:08.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:08 smithi064 ceph-mon[109246]: pgmap v1386: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:46:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:08.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:08.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 109757 109741 92.6707 62.4375 0.00795634 0.0107861 2024-08-17T17:46:08.855 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:08 smithi175 ceph-mon[101434]: pgmap v1386: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:46:08.893 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:08.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:08.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:46:09.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:09.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:09.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:09.200 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:09.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:09.337 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:09.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:09.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:09.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:09.613 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:09.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.764 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:09.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:09.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 110724 110708 92.2409 60.4375 0.0138768 0.0108361 2024-08-17T17:46:09.865 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.906 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:09.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:10.007 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:10.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:10.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:10.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:10.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.294 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:10.294 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:10.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:10.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.432 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:10.500 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:46:10.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:10.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:10.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:10.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:10.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:10.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[104516]: pgmap v1387: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:46:10.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2343169712' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:10.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[109246]: pgmap v1387: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:46:10.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2343169712' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:10.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 111673 111657 91.8077 59.3125 0.00875653 0.0108876 2024-08-17T17:46:10.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:10.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:10 smithi175 ceph-mon[101434]: pgmap v1387: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:46:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2343169712' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:46:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:10.982 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:11.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:46:11.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.036 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:11.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:11.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:11.170 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:11.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:11.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:11.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:11.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:11.459 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.460 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:11.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:11.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:11.588 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:46:11.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:46:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.604 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:11.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:11.658 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:11.705 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:11.737 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:11.769 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:46:11.770 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:11 smithi175 ceph-mon[101434]: osdmap e433: 8 total, 8 up, 7 in 2024-08-17T17:46:11.783 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:46:11.783 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:11 smithi064 ceph-mon[104516]: osdmap e433: 8 total, 8 up, 7 in 2024-08-17T17:46:11.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:46:11.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:11 smithi064 ceph-mon[109246]: osdmap e433: 8 total, 8 up, 7 in 2024-08-17T17:46:11.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 15 112613 112598 91.3791 58.8125 0.00930808 0.0109384 2024-08-17T17:46:11.839 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:11.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:11.974 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:12.001 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:12.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:12.143 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:12.245 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:12.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:12.386 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:46:12.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:12.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:12.422 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:46:12.422 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.427 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:12.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:12.453 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:46:12.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:12.485 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:46:12.485 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:12.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:46:12.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:12.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:12.551 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:46:12.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:12.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:12.594 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:46:12.594 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:12.621 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:46:12.622 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:12.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:12.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:12 smithi064 ceph-mon[109246]: pgmap v1389: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T17:46:12.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4022522775' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:12 smithi064 ceph-mon[104516]: pgmap v1389: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T17:46:12.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4022522775' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:12.696 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:12.798 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:12.813 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 113568 113552 90.972 59.625 0.00781448 0.0109867 2024-08-17T17:46:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.843 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:12.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:12.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:12 smithi175 ceph-mon[101434]: pgmap v1389: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T17:46:12.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4022522775' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:12.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:12.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:13.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:13.122 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:13.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:13.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:13.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:13.388 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:13.463 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:46:13.463 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:46:13.463 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:46:13.463 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:46:13.463 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:46:13.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:13.523 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:13.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:13.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:13.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:13.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:13.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 15 114553 114538 90.6005 61.625 0.0149337 0.0110324 2024-08-17T17:46:13.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:13.935 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:13.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:14.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:14.069 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:14.070 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:14.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:14.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:14.307 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:14.340 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:14.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:13] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.51.0" 2024-08-17T17:46:14.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:14.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.479 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:14.580 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:14.613 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:14.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:14.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:14.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:14 smithi064 ceph-mon[104516]: pgmap v1390: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:46:14.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:14 smithi064 ceph-mon[109246]: pgmap v1390: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:46:14.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2024-08-17T17:46:14.814238+0000 min lat: 0.00139059 max lat: 0.515984 avg lat: 0.0110806 2024-08-17T17:46:14.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:46:14.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 115493 115477 90.2015 58.6875 0.010028 0.0110806 2024-08-17T17:46:14.864 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:14.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:14 smithi175 ceph-mon[101434]: pgmap v1390: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:46:15.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:15.038 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:15.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:15.169 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:15.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:15.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:15.409 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:15.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:15.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:15.587 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:15.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:46:15.689 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:15.733 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:15.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 116466 116450 89.8387 60.8125 0.0107269 0.0111256 2024-08-17T17:46:15.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:15.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:15.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:15.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:16.023 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:16.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:16.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:16.260 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.295 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:16.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:16.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:16.419 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:16.521 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:46:16.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:16.534 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:46:16.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:16.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:16.679 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:16.780 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:16.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 117133 117117 89.2514 41.6875 0.0157454 0.0111991 2024-08-17T17:46:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.824 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:16.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:16.849 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:16 smithi175 ceph-mon[101434]: pgmap v1391: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-08-17T17:46:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4063078458' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[104516]: pgmap v1391: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-08-17T17:46:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4063078458' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[109246]: pgmap v1391: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2024-08-17T17:46:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4063078458' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:16.980 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:17.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:17.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:17.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:17.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:17.352 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:17.386 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:17.487 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:17.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:17.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:17 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:17 smithi175 ceph-mon[101434]: osdmap e434: 8 total, 8 up, 7 in 2024-08-17T17:46:17.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:17.771 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:17.814 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:17.815 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 118115 118099 88.9155 61.375 0.0109077 0.0112412 2024-08-17T17:46:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:17.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[104516]: osdmap e434: 8 total, 8 up, 7 in 2024-08-17T17:46:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:17 smithi064 ceph-mon[109246]: osdmap e434: 8 total, 8 up, 7 in 2024-08-17T17:46:17.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:18.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:18.099 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:18.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.252 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:18.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:18.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:18.422 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:18.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:18.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:18.632 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:18.648 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:18 smithi175 ceph-mon[101434]: pgmap v1393: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:46:18.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:18.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:18 smithi064 ceph-mon[104516]: pgmap v1393: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:46:18.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:18 smithi064 ceph-mon[109246]: pgmap v1393: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:46:18.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:18.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:18.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 119071 119055 88.5683 59.75 0.0146122 0.0112852 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.816 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164890 8.5 8:a88a0acf:::benchmark_data_smithi064_242541_object42397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 534.96179442899995, 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45790435600000001, 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164890 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.817 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850046+0000", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850044+0000", 2024-08-17T17:46:18.818 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850052+0000", 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7350000000000002e-06 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850071+0000", 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8816000000000001e-05 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.819 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850080+0000", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.6379999999999993e-06 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850138+0000", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8700999999999999e-05 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850281+0000", 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000142476 2024-08-17T17:46:18.820 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.850400+0000", 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000118967 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.857056+0000", 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0066560730000000002 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.821 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:23.861294+0000", 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0042383389999999998 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.307933+0000", 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.44663844800000002 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.307950+0000", 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7654000000000001e-05 2024-08-17T17:46:18.822 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.45488.0:164890 8.15 8:a88a0acf:::benchmark_data_smithi064_242541_object42397:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e289)", 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 534.77279331299997, 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.87629330800000005, 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.45488", 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:46:18.823 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 164890 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039047+0000", 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.824 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039042+0000", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999943 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039056+0000", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4637999999999999e-05 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039085+0000", 2024-08-17T17:46:18.825 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8266999999999999e-05 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.039095+0000", 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0818999999999999e-05 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.308685+0000", 2024-08-17T17:46:18.826 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.26958931800000002 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.308694+0000", 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8880000000000003e-06 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.915250+0000", 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.60655672699999996 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.827 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:37:24.915340+0000", 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0210000000000005e-05 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.45488.0:197932 8.7 e310/289 8:e068bd0e:::benchmark_data_smithi064_242541_object75439:head v 310'11359, mlcod=310'11357)", 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 491.06048194599998, 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25881922899999998, 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.828 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751359+0000", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751358+0000", 2024-08-17T17:46:18.829 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:07.751367+0000", 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0070000000000003e-06 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.007946+0000", 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25657962299999998 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.830 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.007961+0000", 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4599e-05 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.008006+0000", 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4879999999999997e-05 2024-08-17T17:46:18.831 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.008025+0000", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8760999999999999e-05 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.010153+0000", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021286819999999998 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:38:08.010178+0000", 2024-08-17T17:46:18.832 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4434999999999999e-05 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:92141 10.a 10:544ba05d:::benchmark_data_smithi064_306376_object92140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:50.295887+0000", 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 28.515953450000001, 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29268362199999998, 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.833 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 92141 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295887+0000", 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295887+0000", 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.834 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295886+0000", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295947+0000", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0881000000000002e-05 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295955+0000", 2024-08-17T17:46:18.835 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.807e-06 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.295962+0000", 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0010000000000004e-06 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.296007+0000", 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5278000000000003e-05 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.836 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.296123+0000", 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000115455 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.296246+0000", 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000123354 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.837 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.300159+0000", 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0039125610000000002 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.308663+0000", 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0085037499999999992 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588463+0000", 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.279799996 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.838 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588516+0000", 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3668999999999997e-05 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588571+0000", 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4401999999999999e-05 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.839 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:92148 10.a 10:5a42e86b:::benchmark_data_smithi064_306376_object92147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:50.303127+0000", 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 28.508713059000002, 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28554005500000001, 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 92148 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.840 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303127+0000", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303127+0000", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303127+0000", 2024-08-17T17:46:18.841 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303160+0000", 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3553999999999998e-05 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303168+0000", 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4930000000000001e-06 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.842 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303174+0000", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6459999999999998e-06 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303206+0000", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2534999999999999e-05 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303296+0000", 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.0006999999999994e-05 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.843 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.303392+0000", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5889000000000003e-05 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.306385+0000", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.002993257 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.308686+0000", 2024-08-17T17:46:18.844 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023003569999999998 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588604+0000", 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.279918 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588638+0000", 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4495000000000001e-05 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.845 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588667+0000", 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9329e-05 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:92172 10.a 10:54f30f5f:::benchmark_data_smithi064_306376_object92171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-08-17T17:46:18.846 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:50.328395+0000", 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 28.483445161999999, 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.26037031199999999, 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 92172 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.847 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328395+0000", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328395+0000", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328394+0000", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.848 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328462+0000", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.7545000000000005e-05 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328472+0000", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0844e-05 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328476+0000", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.0090000000000001e-06 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.849 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328503+0000", 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6968e-05 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328594+0000", 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1039999999999996e-05 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.328683+0000", 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8621000000000004e-05 2024-08-17T17:46:18.850 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.331038+0000", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0023549059999999999 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.331441+0000", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00040357199999999999 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588696+0000", 2024-08-17T17:46:18.851 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25725443199999998 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588737+0000", 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1427999999999997e-05 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:50.588766+0000", 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8266999999999999e-05 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.852 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:99645 10.f 10:fcd23f12:::benchmark_data_smithi064_306376_object99644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:57.813616+0000", 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 20.998224382, 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19139825799999999, 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 99645 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.853 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813616+0000", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813616+0000", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813614+0000", 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:46:18.854 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813707+0000", 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.2881999999999996e-05 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813723+0000", 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5614e-05 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813731+0000", 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1620000000000008e-06 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813791+0000", 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0702999999999998e-05 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.813931+0000", 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013965700000000001 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.856 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.814053+0000", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012233599999999999 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.817559+0000", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035051660000000001 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.818360+0000", 2024-08-17T17:46:18.857 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00080183800000000003 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.004674+0000", 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18631370999999999 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.004872+0000", 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000197362 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.858 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.005014+0000", 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014285200000000001 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:99662 10.0 10:0366675e:::benchmark_data_smithi064_306376_object99661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:57.826008+0000", 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 20.985832432999999, 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.190974216, 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.859 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 99662 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826008+0000", 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.860 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826008+0000", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826007+0000", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826110+0000", 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010324599999999999 2024-08-17T17:46:18.861 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826120+0000", 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0169e-05 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826125+0000", 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0509999999999996e-06 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.862 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826155+0000", 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9986000000000001e-05 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826291+0000", 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013601800000000001 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.826427+0000", 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013583000000000001 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.863 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.829765+0000", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003338209 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:57.830755+0000", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00098987199999999993 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.016888+0000", 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.186132239 2024-08-17T17:46:18.864 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.016943+0000", 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5143999999999999e-05 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:58.016982+0000", 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9561000000000003e-05 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.865 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:100938 10.f 10:f942557e:::benchmark_data_smithi064_306376_object100937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:59.379749+0000", 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 19.432091541999998, 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25020018799999999, 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 100938 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.866 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379749+0000", 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379749+0000", 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.867 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379748+0000", 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379801+0000", 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2701000000000002e-05 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379812+0000", 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0759e-05 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.868 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.379817+0000", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8839999999999999e-06 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.383754+0000", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0039361040000000002 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.383922+0000", 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016882600000000001 2024-08-17T17:46:18.869 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.384094+0000", 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017122799999999999 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.387600+0000", 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035061710000000002 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.870 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.387786+0000", 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018656599999999999 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629834+0000", 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.24204804099999999 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629905+0000", 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0535000000000001e-05 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.871 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629949+0000", 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4177000000000001e-05 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:100948 10.e e430/410 10:7b02af26:::benchmark_data_smithi064_306376_object100947:head v 430'6392, mlcod=430'6390)", 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:59.394455+0000", 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 19.417385188000001, 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.238166397, 2024-08-17T17:46:18.872 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394455+0000", 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394455+0000", 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.873 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394454+0000", 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.874 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394481+0000", 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6431000000000002e-05 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394494+0000", 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3332e-05 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.875 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394501+0000", 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3780000000000003e-06 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394545+0000", 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.392e-05 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.876 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.394565+0000", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9847000000000002e-05 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.632546+0000", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23798055200000001 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.632622+0000", 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5987000000000003e-05 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.877 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:100954 10.a 10:574f5739:::benchmark_data_smithi064_306376_object100953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:59.397040+0000", 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 19.414800400000001, 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.246137421, 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.878 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 100954 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397040+0000", 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397040+0000", 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.879 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397038+0000", 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397074+0000", 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5815000000000002e-05 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.880 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397089+0000", 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4802e-05 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397097+0000", 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7379999999999994e-06 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397144+0000", 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7441999999999997e-05 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.881 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397270+0000", 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012621799999999999 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629075+0000", 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23180465 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.882 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.632571+0000", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0034955810000000002 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.640017+0000", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0074463519999999998 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643085+0000", 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030679840000000002 2024-08-17T17:46:18.883 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643143+0000", 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8044999999999997e-05 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643177+0000", 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4434000000000003e-05 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.884 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:100950 10.d e430/410 10:b4d64595:::benchmark_data_smithi064_306376_object100949:head v 430'6279, mlcod=430'6276)", 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:59.397444+0000", 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 19.414396177, 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.245205232, 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397444+0000", 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.885 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397444+0000", 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397443+0000", 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.886 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.397481+0000", 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7540000000000003e-05 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.628927+0000", 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23144563600000001 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.628941+0000", 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4623999999999999e-05 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.887 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.628988+0000", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7126000000000002e-05 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629013+0000", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5023000000000001e-05 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.641273+0000", 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.012259772 2024-08-17T17:46:18.888 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.642650+0000", 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001376424 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:100961 10.f 10:f34835ee:::benchmark_data_smithi064_306376_object100960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:59.405263+0000", 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 19.406577276, 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23861074199999999, 2024-08-17T17:46:18.889 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 100961 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.405263+0000", 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.890 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.405263+0000", 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.405261+0000", 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.891 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.405318+0000", 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.6780999999999997e-05 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629334+0000", 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.22401655700000001 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629344+0000", 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0056e-05 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.892 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.629978+0000", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00063374600000000003 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.630153+0000", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017458799999999999 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.630313+0000", 2024-08-17T17:46:18.893 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016054700000000001 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.640148+0000", 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0098344639999999994 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643679+0000", 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035312820000000002 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.894 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643687+0000", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9820000000000008e-06 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643838+0000", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015095700000000001 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:59.643874+0000", 2024-08-17T17:46:18.895 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5948999999999998e-05 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:114794 10.a 10:50b16c32:::benchmark_data_smithi064_306376_object114793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:14.039653+0000", 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.7721878630000001, 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.088750700000000002, 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.896 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 114794 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039653+0000", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039653+0000", 2024-08-17T17:46:18.897 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039652+0000", 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039681+0000", 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.9235999999999999e-05 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.898 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039688+0000", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6549999999999998e-06 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.039693+0000", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6410000000000001e-06 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.043126+0000", 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003433677 2024-08-17T17:46:18.899 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.043252+0000", 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012596800000000001 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.043386+0000", 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013403500000000001 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.900 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.047174+0000", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037873270000000001 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.057164+0000", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0099900590000000008 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128336+0000", 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.071172279000000005 2024-08-17T17:46:18.901 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128375+0000", 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8813000000000003e-05 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128403+0000", 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.8428999999999998e-05 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.902 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:114797 10.f 10:ff30fd46:::benchmark_data_smithi064_306376_object114796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:14.042238+0000", 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.7696028779999997, 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.089297566999999994, 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 114797 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.903 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042238+0000", 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042238+0000", 2024-08-17T17:46:18.904 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.905 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:18.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:18.906 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.906 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042236+0000", 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042365+0000", 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012838300000000001 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.907 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042378+0000", 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3621999999999999e-05 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.042385+0000", 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5560000000000004e-06 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.046747+0000", 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0043618299999999997 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.908 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.046922+0000", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017594999999999999 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.047074+0000", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015134600000000001 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.057118+0000", 2024-08-17T17:46:18.909 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010044371999999999 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.058661+0000", 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.001543242 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.131407+0000", 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.072745545999999994 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.910 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.131496+0000", 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8864999999999997e-05 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.131535+0000", 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9298000000000002e-05 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:114802 10.7 e433/413 10:eccb214d:::benchmark_data_smithi064_306376_object114801:head v 433'7096, mlcod=433'7093)", 2024-08-17T17:46:18.911 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:14.048958+0000", 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.7628826789999996, 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.090809560999999997, 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048958+0000", 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.912 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048958+0000", 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048958+0000", 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048979+0000", 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1449000000000001e-05 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.913 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048985+0000", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8510000000000004e-06 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.048989+0000", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.9890000000000003e-06 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128037+0000", 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.079048194000000002 2024-08-17T17:46:18.914 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128058+0000", 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1327000000000001e-05 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.139637+0000", 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.011578418 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.915 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.139767+0000", 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013059500000000001 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:114808 10.f 10:f06f1733:::benchmark_data_smithi064_306376_object114807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:14.056047+0000", 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 4.7557931099999999, 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.092040995, 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.916 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 114808 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056047+0000", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056047+0000", 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.917 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056045+0000", 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056083+0000", 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7827000000000002e-05 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.918 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056101+0000", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8238999999999999e-05 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.056110+0000", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.3449999999999995e-06 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.058675+0000", 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0025644579999999999 2024-08-17T17:46:18.919 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.058777+0000", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010272099999999999 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.128065+0000", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.069287692999999997 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.131656+0000", 2024-08-17T17:46:18.920 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035905500000000001 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.139635+0000", 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0079796759999999998 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.147945+0000", 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0083092500000000007 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.921 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.148045+0000", 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000100467 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:14.148088+0000", 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3339e-05 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.922 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:116902 10.f 10:feb1f0e7:::benchmark_data_smithi064_306376_object116901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:16.298574+0000", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 2.5132660539999998, 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31764725900000002, 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 116902 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.923 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298574+0000", 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298574+0000", 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298573+0000", 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:18.924 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.926 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.926 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.926 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298610+0000", 2024-08-17T17:46:18.926 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7024999999999999e-05 2024-08-17T17:46:18.926 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298627+0000", 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6458999999999999e-05 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298633+0000", 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4110000000000001e-06 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.927 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298670+0000", 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7212000000000002e-05 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.298770+0000", 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.9767999999999998e-05 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.299042+0000", 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027153600000000001 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.928 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303010+0000", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0039681350000000002 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.307518+0000", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0045086370000000002 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.616100+0000", 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.30858177599999997 2024-08-17T17:46:18.929 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.616184+0000", 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4118999999999999e-05 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.616222+0000", 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7486e-05 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.930 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:116905 10.f 10:f4e7edac:::benchmark_data_smithi064_306376_object116904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:16.302886+0000", 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 2.5089543729999999, 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31674892100000002, 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 116905 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.931 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.302886+0000", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.302886+0000", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.302885+0000", 2024-08-17T17:46:18.932 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.302993+0000", 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010874099999999999 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303007+0000", 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3472000000000001e-05 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.933 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303014+0000", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.754e-06 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303052+0000", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8708999999999998e-05 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303284+0000", 2024-08-17T17:46:18.934 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023197800000000001 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.303998+0000", 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00071377400000000003 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.307548+0000", 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0035499590000000001 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.935 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.307700+0000", 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000151719 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.619547+0000", 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31184728900000003 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.619599+0000", 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1579999999999997e-05 2024-08-17T17:46:18.936 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.619635+0000", 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6405000000000001e-05 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:116913 10.f 10:f3c43182:::benchmark_data_smithi064_306376_object116912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:16.311822+0000", 2024-08-17T17:46:18.937 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 2.5000183310000001, 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31624002400000001, 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 116913 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311822+0000", 2024-08-17T17:46:18.938 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311822+0000", 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311821+0000", 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:46:18.939 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311888+0000", 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6459999999999997e-05 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311902+0000", 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4460999999999999e-05 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311908+0000", 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.1179999999999997e-06 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.940 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.311967+0000", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8365000000000001e-05 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.312105+0000", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013835400000000001 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.312259+0000", 2024-08-17T17:46:18.941 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015385799999999999 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.315651+0000", 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0033917309999999998 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.316586+0000", 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00093543999999999995 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:18.942 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.621975+0000", 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.305388414 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.622018+0000", 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3077000000000001e-05 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:16.628062+0000", 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0060444660000000001 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.943 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.944 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.944 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:46:18.944 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:46:18.944 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:46:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:46:18.948 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:46:19.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:19.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:19.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:19.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:19.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:19.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:19.312 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:19.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:19.448 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:19.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:19.586 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:19.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:19.730 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:19.814 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 120050 120034 88.2461 61.1875 0.0191247 0.0113266 2024-08-17T17:46:19.832 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:19.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:19.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:19 smithi175 ceph-mon[101434]: from='client.36189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:19.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:19 smithi064 ceph-mon[104516]: from='client.36189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:19.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:19 smithi064 ceph-mon[109246]: from='client.36189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:19.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:19.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:20.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:46:20.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:46:20.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:20.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:20.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:20.266 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:20.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:20.368 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:20.413 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:20.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:20.554 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:20.627 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1393,"stamp":"2024-08-17T17:46:17.369791+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7632721893,"num_objects":116421,"num_object_clones":0,"num_object_copies":349263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116421,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":117301,"num_write_kb":7461777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8657757,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111965,"ondisk_log_size":111965,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":28206544,"kb_used_data":22604632,"kb_used_omap":40,"kb_used_meta":5601815,"kb_avail":628009520,"statfs":{"total":671965249536,"available":643081748480,"internally_reserved":0,"allocated":23147143168,"data_stored":23132447086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41814,"internal_metadata":5736258730},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,5,9,8,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":614662144,"num_objects":9379,"num_object_clones":0,"num_object_copies":28137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9379,"num_write_kb":600256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001968"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603751+0000","last_change":"2024-08-17T17:44:45.659658+0000","last_active":"2024-08-17T17:46:16.603751+0000","last_peered":"2024-08-17T17:46:16.603751+0000","last_clean":"2024-08-17T17:46:16.603751+0000","last_became_active":"2024-08-17T17:44:45.659189+0000","last_became_peered":"2024-08-17T17:44:45.659189+0000","last_unstale":"2024-08-17T17:46:16.603751+0000","last_undegraded":"2024-08-17T17:46:16.603751+0000","last_fullsized":"2024-08-17T17:46:16.603751+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:45.422590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":435,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214323+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:46:05.214323+0000","last_peered":"2024-08-17T17:46:05.214323+0000","last_clean":"2024-08-17T17:46:05.214323+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:46:05.214323+0000","last_undegraded":"2024-08-17T17:46:05.214323+0000","last_fullsized":"2024-08-17T17:46:05.214323+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199761+0000","last_change":"2024-08-17T17:44:45.650591+0000","last_active":"2024-08-17T17:46:05.199761+0000","last_peered":"2024-08-17T17:46:05.199761+0000","last_clean":"2024-08-17T17:46:05.199761+0000","last_became_active":"2024-08-17T17:44:45.650249+0000","last_became_peered":"2024-08-17T17:44:45.650249+0000","last_unstale":"2024-08-17T17:46:05.199761+0000","last_undegraded":"2024-08-17T17:46:05.199761+0000","last_fullsized":"2024-08-17T17:46:05.199761+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:06.858698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":727,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506583+0000","last_change":"2024-08-17T17:44:45.663771+0000","last_active":"2024-08-17T17:46:10.506583+0000","last_peered":"2024-08-17T17:46:10.506583+0000","last_clean":"2024-08-17T17:46:10.506583+0000","last_became_active":"2024-08-17T17:44:45.663321+0000","last_became_peered":"2024-08-17T17:44:45.663321+0000","last_unstale":"2024-08-17T17:46:10.506583+0000","last_undegraded":"2024-08-17T17:46:10.506583+0000","last_fullsized":"2024-08-17T17:46:10.506583+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:30.374220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":726,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506815+0000","last_change":"2024-08-17T17:44:45.657665+0000","last_active":"2024-08-17T17:46:10.506815+0000","last_peered":"2024-08-17T17:46:10.506815+0000","last_clean":"2024-08-17T17:46:10.506815+0000","last_became_active":"2024-08-17T17:44:45.657342+0000","last_became_peered":"2024-08-17T17:44:45.657342+0000","last_unstale":"2024-08-17T17:46:10.506815+0000","last_undegraded":"2024-08-17T17:46:10.506815+0000","last_fullsized":"2024-08-17T17:46:10.506815+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:00:18.245759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":421,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214611+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:46:05.214611+0000","last_peered":"2024-08-17T17:46:05.214611+0000","last_clean":"2024-08-17T17:46:05.214611+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:46:05.214611+0000","last_undegraded":"2024-08-17T17:46:05.214611+0000","last_fullsized":"2024-08-17T17:46:05.214611+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214998+0000","last_change":"2024-08-17T17:44:45.660525+0000","last_active":"2024-08-17T17:46:05.214998+0000","last_peered":"2024-08-17T17:46:05.214998+0000","last_clean":"2024-08-17T17:46:05.214998+0000","last_became_active":"2024-08-17T17:44:45.660152+0000","last_became_peered":"2024-08-17T17:44:45.660152+0000","last_unstale":"2024-08-17T17:46:05.214998+0000","last_undegraded":"2024-08-17T17:46:05.214998+0000","last_fullsized":"2024-08-17T17:46:05.214998+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:35:09.946483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":369,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.208741+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:46:05.208741+0000","last_peered":"2024-08-17T17:46:05.208741+0000","last_clean":"2024-08-17T17:46:05.208741+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:46:05.208741+0000","last_undegraded":"2024-08-17T17:46:05.208741+0000","last_fullsized":"2024-08-17T17:46:05.208741+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"433'7265","reported_seq":7294,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.547142+0000","last_change":"2024-08-17T17:44:36.608512+0000","last_active":"2024-08-17T17:46:15.547142+0000","last_peered":"2024-08-17T17:46:15.547142+0000","last_clean":"2024-08-17T17:46:15.547142+0000","last_became_active":"2024-08-17T17:44:36.608224+0000","last_became_peered":"2024-08-17T17:44:36.608224+0000","last_unstale":"2024-08-17T17:46:15.547142+0000","last_undegraded":"2024-08-17T17:46:15.547142+0000","last_fullsized":"2024-08-17T17:46:15.547142+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7265,"log_dups_size":0,"ondisk_log_size":7265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:46.388975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476119040,"num_objects":7265,"num_object_clones":0,"num_object_copies":21795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7265,"num_write_kb":464960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":430,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.215144+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:46:05.215144+0000","last_peered":"2024-08-17T17:46:05.215144+0000","last_clean":"2024-08-17T17:46:05.215144+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:46:05.215144+0000","last_undegraded":"2024-08-17T17:46:05.215144+0000","last_fullsized":"2024-08-17T17:46:05.215144+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":228,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199118+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:46:05.199118+0000","last_peered":"2024-08-17T17:46:05.199118+0000","last_clean":"2024-08-17T17:46:05.199118+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:46:05.199118+0000","last_undegraded":"2024-08-17T17:46:05.199118+0000","last_fullsized":"2024-08-17T17:46:05.199118+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"433'7168","reported_seq":7199,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.546908+0000","last_change":"2024-08-17T17:44:45.652247+0000","last_active":"2024-08-17T17:46:15.546908+0000","last_peered":"2024-08-17T17:46:15.546908+0000","last_clean":"2024-08-17T17:46:15.546908+0000","last_became_active":"2024-08-17T17:44:45.651926+0000","last_became_peered":"2024-08-17T17:44:45.651926+0000","last_unstale":"2024-08-17T17:46:15.546908+0000","last_undegraded":"2024-08-17T17:46:15.546908+0000","last_fullsized":"2024-08-17T17:46:15.546908+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7168,"log_dups_size":0,"ondisk_log_size":7168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:39.517297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":469762048,"num_objects":7168,"num_object_clones":0,"num_object_copies":21504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7168,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":432,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.015812+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:46:06.015812+0000","last_peered":"2024-08-17T17:46:06.015812+0000","last_clean":"2024-08-17T17:46:06.015812+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:46:06.015812+0000","last_undegraded":"2024-08-17T17:46:06.015812+0000","last_fullsized":"2024-08-17T17:46:06.015812+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":336,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.508094+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:10.508094+0000","last_peered":"2024-08-17T17:46:10.508094+0000","last_clean":"2024-08-17T17:46:10.508094+0000","last_became_active":"2024-08-17T17:44:45.658981+0000","last_became_peered":"2024-08-17T17:44:45.658981+0000","last_unstale":"2024-08-17T17:46:10.508094+0000","last_undegraded":"2024-08-17T17:46:10.508094+0000","last_fullsized":"2024-08-17T17:46:10.508094+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:59.247763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"433'7290","reported_seq":7354,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.320279+0000","last_change":"2024-08-17T17:44:45.662022+0000","last_active":"2024-08-17T17:46:16.320279+0000","last_peered":"2024-08-17T17:46:16.320279+0000","last_clean":"2024-08-17T17:46:16.320279+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:16.320279+0000","last_undegraded":"2024-08-17T17:46:16.320279+0000","last_fullsized":"2024-08-17T17:46:16.320279+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7290,"log_dups_size":0,"ondisk_log_size":7290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:02.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477757440,"num_objects":7290,"num_object_clones":0,"num_object_copies":21870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7290,"num_write_kb":466560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":449,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.015781+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:46:06.015781+0000","last_peered":"2024-08-17T17:46:06.015781+0000","last_clean":"2024-08-17T17:46:06.015781+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:46:06.015781+0000","last_undegraded":"2024-08-17T17:46:06.015781+0000","last_fullsized":"2024-08-17T17:46:06.015781+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":349,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.207863+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:46:05.207863+0000","last_peered":"2024-08-17T17:46:05.207863+0000","last_clean":"2024-08-17T17:46:05.207863+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:46:05.207863+0000","last_undegraded":"2024-08-17T17:46:05.207863+0000","last_fullsized":"2024-08-17T17:46:05.207863+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"433'7329","reported_seq":7393,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.255818+0000","last_change":"2024-08-17T17:44:45.658418+0000","last_active":"2024-08-17T17:46:16.255818+0000","last_peered":"2024-08-17T17:46:16.255818+0000","last_clean":"2024-08-17T17:46:16.255818+0000","last_became_active":"2024-08-17T17:44:45.658105+0000","last_became_peered":"2024-08-17T17:44:45.658105+0000","last_unstale":"2024-08-17T17:46:16.255818+0000","last_undegraded":"2024-08-17T17:46:16.255818+0000","last_fullsized":"2024-08-17T17:46:16.255818+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:47.860621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480313344,"num_objects":7329,"num_object_clones":0,"num_object_copies":21987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7329,"num_write_kb":469056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":237,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199737+0000","last_change":"2024-08-17T17:44:45.657383+0000","last_active":"2024-08-17T17:46:05.199737+0000","last_peered":"2024-08-17T17:46:05.199737+0000","last_clean":"2024-08-17T17:46:05.199737+0000","last_became_active":"2024-08-17T17:44:45.656436+0000","last_became_peered":"2024-08-17T17:44:45.656436+0000","last_unstale":"2024-08-17T17:46:05.199737+0000","last_undegraded":"2024-08-17T17:46:05.199737+0000","last_fullsized":"2024-08-17T17:46:05.199737+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:23:40.172334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":741,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507257+0000","last_change":"2024-08-17T17:45:27.582532+0000","last_active":"2024-08-17T17:46:10.507257+0000","last_peered":"2024-08-17T17:46:10.507257+0000","last_clean":"2024-08-17T17:46:10.507257+0000","last_became_active":"2024-08-17T17:45:27.582184+0000","last_became_peered":"2024-08-17T17:45:27.582184+0000","last_unstale":"2024-08-17T17:46:10.507257+0000","last_undegraded":"2024-08-17T17:46:10.507257+0000","last_fullsized":"2024-08-17T17:46:10.507257+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:28.064920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"434'7286","reported_seq":7345,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.612109+0000","last_change":"2024-08-17T17:44:36.607826+0000","last_active":"2024-08-17T17:46:16.612109+0000","last_peered":"2024-08-17T17:46:16.612109+0000","last_clean":"2024-08-17T17:46:16.612109+0000","last_became_active":"2024-08-17T17:44:36.607543+0000","last_became_peered":"2024-08-17T17:44:36.607543+0000","last_unstale":"2024-08-17T17:46:16.612109+0000","last_undegraded":"2024-08-17T17:46:16.612109+0000","last_fullsized":"2024-08-17T17:46:16.612109+0000","mapping_epoch":410,"log_start":"418'400","ondisk_log_start":"418'400","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6886,"log_dups_size":0,"ondisk_log_size":6886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:22.366195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477495296,"num_objects":7286,"num_object_clones":0,"num_object_copies":21858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7286,"num_write_kb":466304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":807,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.015714+0000","last_change":"2024-08-17T17:44:48.661664+0000","last_active":"2024-08-17T17:46:06.015714+0000","last_peered":"2024-08-17T17:46:06.015714+0000","last_clean":"2024-08-17T17:46:06.015714+0000","last_became_active":"2024-08-17T17:44:48.661277+0000","last_became_peered":"2024-08-17T17:44:48.661277+0000","last_unstale":"2024-08-17T17:46:06.015714+0000","last_undegraded":"2024-08-17T17:46:06.015714+0000","last_fullsized":"2024-08-17T17:46:06.015714+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.305103+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":77,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506756+0000","last_change":"2024-08-17T17:44:45.670332+0000","last_active":"2024-08-17T17:46:10.506756+0000","last_peered":"2024-08-17T17:46:10.506756+0000","last_clean":"2024-08-17T17:46:10.506756+0000","last_became_active":"2024-08-17T17:44:45.660508+0000","last_became_peered":"2024-08-17T17:44:45.660508+0000","last_unstale":"2024-08-17T17:46:10.506756+0000","last_undegraded":"2024-08-17T17:46:10.506756+0000","last_fullsized":"2024-08-17T17:46:10.506756+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:17.489964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":359,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.210133+0000","last_change":"2024-08-17T17:44:45.656521+0000","last_active":"2024-08-17T17:46:05.210133+0000","last_peered":"2024-08-17T17:46:05.210133+0000","last_clean":"2024-08-17T17:46:05.210133+0000","last_became_active":"2024-08-17T17:44:45.656178+0000","last_became_peered":"2024-08-17T17:44:45.656178+0000","last_unstale":"2024-08-17T17:46:05.210133+0000","last_undegraded":"2024-08-17T17:46:05.210133+0000","last_fullsized":"2024-08-17T17:46:05.210133+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:48.486263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"433'7257","reported_seq":7298,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.581013+0000","last_change":"2024-08-17T17:44:45.653325+0000","last_active":"2024-08-17T17:46:15.581013+0000","last_peered":"2024-08-17T17:46:15.581013+0000","last_clean":"2024-08-17T17:46:15.581013+0000","last_became_active":"2024-08-17T17:44:45.652943+0000","last_became_peered":"2024-08-17T17:44:45.652943+0000","last_unstale":"2024-08-17T17:46:15.581013+0000","last_undegraded":"2024-08-17T17:46:15.581013+0000","last_fullsized":"2024-08-17T17:46:15.581013+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7257,"log_dups_size":0,"ondisk_log_size":7257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:29:49.759034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475594752,"num_objects":7257,"num_object_clones":0,"num_object_copies":21771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7257,"num_write_kb":464448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":471,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.501778+0000","last_change":"2024-08-17T17:44:48.661693+0000","last_active":"2024-08-17T17:46:10.501778+0000","last_peered":"2024-08-17T17:46:10.501778+0000","last_clean":"2024-08-17T17:46:10.501778+0000","last_became_active":"2024-08-17T17:44:48.661280+0000","last_became_peered":"2024-08-17T17:44:48.661280+0000","last_unstale":"2024-08-17T17:46:10.501778+0000","last_undegraded":"2024-08-17T17:46:10.501778+0000","last_fullsized":"2024-08-17T17:46:10.501778+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:56.378196+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209521+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:46:05.209521+0000","last_peered":"2024-08-17T17:46:05.209521+0000","last_clean":"2024-08-17T17:46:05.209521+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:46:05.209521+0000","last_undegraded":"2024-08-17T17:46:05.209521+0000","last_fullsized":"2024-08-17T17:46:05.209521+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":326,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.210196+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:46:05.210196+0000","last_peered":"2024-08-17T17:46:05.210196+0000","last_clean":"2024-08-17T17:46:05.210196+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:46:05.210196+0000","last_undegraded":"2024-08-17T17:46:05.210196+0000","last_fullsized":"2024-08-17T17:46:05.210196+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"433'7292","reported_seq":7324,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:14.808489+0000","last_change":"2024-08-17T17:44:36.610056+0000","last_active":"2024-08-17T17:46:14.808489+0000","last_peered":"2024-08-17T17:46:14.808489+0000","last_clean":"2024-08-17T17:46:14.808489+0000","last_became_active":"2024-08-17T17:44:36.609357+0000","last_became_peered":"2024-08-17T17:44:36.609357+0000","last_unstale":"2024-08-17T17:46:14.808489+0000","last_undegraded":"2024-08-17T17:46:14.808489+0000","last_fullsized":"2024-08-17T17:46:14.808489+0000","mapping_epoch":410,"log_start":"418'1000","ondisk_log_start":"418'1000","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:01.344940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477888512,"num_objects":7292,"num_object_clones":0,"num_object_copies":21876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7292,"num_write_kb":466688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":327,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507880+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:46:10.507880+0000","last_peered":"2024-08-17T17:46:10.507880+0000","last_clean":"2024-08-17T17:46:10.507880+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:46:10.507880+0000","last_undegraded":"2024-08-17T17:46:10.507880+0000","last_fullsized":"2024-08-17T17:46:10.507880+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":341,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.215262+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:46:05.215262+0000","last_peered":"2024-08-17T17:46:05.215262+0000","last_clean":"2024-08-17T17:46:05.215262+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:46:05.215262+0000","last_undegraded":"2024-08-17T17:46:05.215262+0000","last_fullsized":"2024-08-17T17:46:05.215262+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"433'7194","reported_seq":7226,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.991413+0000","last_change":"2024-08-17T17:44:36.610369+0000","last_active":"2024-08-17T17:46:15.991413+0000","last_peered":"2024-08-17T17:46:15.991413+0000","last_clean":"2024-08-17T17:46:15.991413+0000","last_became_active":"2024-08-17T17:44:36.610096+0000","last_became_peered":"2024-08-17T17:44:36.610096+0000","last_unstale":"2024-08-17T17:46:15.991413+0000","last_undegraded":"2024-08-17T17:46:15.991413+0000","last_fullsized":"2024-08-17T17:46:15.991413+0000","mapping_epoch":410,"log_start":"418'1000","ondisk_log_start":"418'1000","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:28.841902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":471465984,"num_objects":7194,"num_object_clones":0,"num_object_copies":21582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":460416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":450,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502802+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:46:10.502802+0000","last_peered":"2024-08-17T17:46:10.502802+0000","last_clean":"2024-08-17T17:46:10.502802+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:46:10.502802+0000","last_undegraded":"2024-08-17T17:46:10.502802+0000","last_fullsized":"2024-08-17T17:46:10.502802+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":116,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603229+0000","last_change":"2024-08-17T17:44:45.657126+0000","last_active":"2024-08-17T17:46:16.603229+0000","last_peered":"2024-08-17T17:46:16.603229+0000","last_clean":"2024-08-17T17:46:16.603229+0000","last_became_active":"2024-08-17T17:44:45.656799+0000","last_became_peered":"2024-08-17T17:44:45.656799+0000","last_unstale":"2024-08-17T17:46:16.603229+0000","last_undegraded":"2024-08-17T17:46:16.603229+0000","last_fullsized":"2024-08-17T17:46:16.603229+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:06.941588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":725,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.512534+0000","last_change":"2024-08-17T17:44:45.659396+0000","last_active":"2024-08-17T17:46:10.512534+0000","last_peered":"2024-08-17T17:46:10.512534+0000","last_clean":"2024-08-17T17:46:10.512534+0000","last_became_active":"2024-08-17T17:44:45.659081+0000","last_became_peered":"2024-08-17T17:44:45.659081+0000","last_unstale":"2024-08-17T17:46:10.512534+0000","last_undegraded":"2024-08-17T17:46:10.512534+0000","last_fullsized":"2024-08-17T17:46:10.512534+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:14.819296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"434'7319","reported_seq":7377,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603501+0000","last_change":"2024-08-17T17:44:36.610395+0000","last_active":"2024-08-17T17:46:16.603501+0000","last_peered":"2024-08-17T17:46:16.603501+0000","last_clean":"2024-08-17T17:46:16.603501+0000","last_became_active":"2024-08-17T17:44:36.610060+0000","last_became_peered":"2024-08-17T17:44:36.610060+0000","last_unstale":"2024-08-17T17:46:16.603501+0000","last_undegraded":"2024-08-17T17:46:16.603501+0000","last_fullsized":"2024-08-17T17:46:16.603501+0000","mapping_epoch":410,"log_start":"418'500","ondisk_log_start":"418'500","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:23.272119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479657984,"num_objects":7319,"num_object_clones":0,"num_object_copies":21957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7319,"num_write_kb":468416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":774,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.015695+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:46:06.015695+0000","last_peered":"2024-08-17T17:46:06.015695+0000","last_clean":"2024-08-17T17:46:06.015695+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:46:06.015695+0000","last_undegraded":"2024-08-17T17:46:06.015695+0000","last_fullsized":"2024-08-17T17:46:06.015695+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1027,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199911+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:46:05.199911+0000","last_peered":"2024-08-17T17:46:05.199911+0000","last_clean":"2024-08-17T17:46:05.199911+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:46:05.199911+0000","last_undegraded":"2024-08-17T17:46:05.199911+0000","last_fullsized":"2024-08-17T17:46:05.199911+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":360,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199291+0000","last_change":"2024-08-17T17:44:45.651175+0000","last_active":"2024-08-17T17:46:05.199291+0000","last_peered":"2024-08-17T17:46:05.199291+0000","last_clean":"2024-08-17T17:46:05.199291+0000","last_became_active":"2024-08-17T17:44:45.650847+0000","last_became_peered":"2024-08-17T17:44:45.650847+0000","last_unstale":"2024-08-17T17:46:05.199291+0000","last_undegraded":"2024-08-17T17:46:05.199291+0000","last_fullsized":"2024-08-17T17:46:05.199291+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:30.892351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":340,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209823+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:46:05.209823+0000","last_peered":"2024-08-17T17:46:05.209823+0000","last_clean":"2024-08-17T17:46:05.209823+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:46:05.209823+0000","last_undegraded":"2024-08-17T17:46:05.209823+0000","last_fullsized":"2024-08-17T17:46:05.209823+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"433'7239","reported_seq":7296,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.256479+0000","last_change":"2024-08-17T17:44:36.968089+0000","last_active":"2024-08-17T17:46:16.256479+0000","last_peered":"2024-08-17T17:46:16.256479+0000","last_clean":"2024-08-17T17:46:16.256479+0000","last_became_active":"2024-08-17T17:44:36.967095+0000","last_became_peered":"2024-08-17T17:44:36.967095+0000","last_unstale":"2024-08-17T17:46:16.256479+0000","last_undegraded":"2024-08-17T17:46:16.256479+0000","last_fullsized":"2024-08-17T17:46:16.256479+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:57.468862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474415104,"num_objects":7239,"num_object_clones":0,"num_object_copies":21717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7239,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":533,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603562+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:46:16.603562+0000","last_peered":"2024-08-17T17:46:16.603562+0000","last_clean":"2024-08-17T17:46:16.603562+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:46:16.603562+0000","last_undegraded":"2024-08-17T17:46:16.603562+0000","last_fullsized":"2024-08-17T17:46:16.603562+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":814,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214200+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:46:05.214200+0000","last_peered":"2024-08-17T17:46:05.214200+0000","last_clean":"2024-08-17T17:46:05.214200+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:46:05.214200+0000","last_undegraded":"2024-08-17T17:46:05.214200+0000","last_fullsized":"2024-08-17T17:46:05.214200+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":742,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507825+0000","last_change":"2024-08-17T17:44:45.659724+0000","last_active":"2024-08-17T17:46:10.507825+0000","last_peered":"2024-08-17T17:46:10.507825+0000","last_clean":"2024-08-17T17:46:10.507825+0000","last_became_active":"2024-08-17T17:44:45.654153+0000","last_became_peered":"2024-08-17T17:44:45.654153+0000","last_unstale":"2024-08-17T17:46:10.507825+0000","last_undegraded":"2024-08-17T17:46:10.507825+0000","last_fullsized":"2024-08-17T17:46:10.507825+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.165904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":732,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506577+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:46:10.506577+0000","last_peered":"2024-08-17T17:46:10.506577+0000","last_clean":"2024-08-17T17:46:10.506577+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:46:10.506577+0000","last_undegraded":"2024-08-17T17:46:10.506577+0000","last_fullsized":"2024-08-17T17:46:10.506577+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"433'7288","reported_seq":7320,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.517726+0000","last_change":"2024-08-17T17:44:36.966264+0000","last_active":"2024-08-17T17:46:15.517726+0000","last_peered":"2024-08-17T17:46:15.517726+0000","last_clean":"2024-08-17T17:46:15.517726+0000","last_became_active":"2024-08-17T17:44:36.965940+0000","last_became_peered":"2024-08-17T17:44:36.965940+0000","last_unstale":"2024-08-17T17:46:15.517726+0000","last_undegraded":"2024-08-17T17:46:15.517726+0000","last_fullsized":"2024-08-17T17:46:15.517726+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7288,"log_dups_size":0,"ondisk_log_size":7288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:16.157942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477626368,"num_objects":7288,"num_object_clones":0,"num_object_copies":21864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7288,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":499,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214067+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:46:05.214067+0000","last_peered":"2024-08-17T17:46:05.214067+0000","last_clean":"2024-08-17T17:46:05.214067+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:46:05.214067+0000","last_undegraded":"2024-08-17T17:46:05.214067+0000","last_fullsized":"2024-08-17T17:46:05.214067+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":478,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507932+0000","last_change":"2024-08-17T17:44:45.679401+0000","last_active":"2024-08-17T17:46:10.507932+0000","last_peered":"2024-08-17T17:46:10.507932+0000","last_clean":"2024-08-17T17:46:10.507932+0000","last_became_active":"2024-08-17T17:44:45.664192+0000","last_became_peered":"2024-08-17T17:44:45.664192+0000","last_unstale":"2024-08-17T17:46:10.507932+0000","last_undegraded":"2024-08-17T17:46:10.507932+0000","last_fullsized":"2024-08-17T17:46:10.507932+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:21.066280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":727,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603312+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:46:16.603312+0000","last_peered":"2024-08-17T17:46:16.603312+0000","last_clean":"2024-08-17T17:46:16.603312+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:46:16.603312+0000","last_undegraded":"2024-08-17T17:46:16.603312+0000","last_fullsized":"2024-08-17T17:46:16.603312+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":774,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.604321+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:46:16.604321+0000","last_peered":"2024-08-17T17:46:16.604321+0000","last_clean":"2024-08-17T17:46:16.604321+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:46:16.604321+0000","last_undegraded":"2024-08-17T17:46:16.604321+0000","last_fullsized":"2024-08-17T17:46:16.604321+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"433'7309","reported_seq":7341,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:14.770502+0000","last_change":"2024-08-17T17:44:36.610536+0000","last_active":"2024-08-17T17:46:14.770502+0000","last_peered":"2024-08-17T17:46:14.770502+0000","last_clean":"2024-08-17T17:46:14.770502+0000","last_became_active":"2024-08-17T17:44:36.610195+0000","last_became_peered":"2024-08-17T17:44:36.610195+0000","last_unstale":"2024-08-17T17:46:14.770502+0000","last_undegraded":"2024-08-17T17:46:14.770502+0000","last_fullsized":"2024-08-17T17:46:14.770502+0000","mapping_epoch":410,"log_start":"418'1000","ondisk_log_start":"418'1000","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6309,"log_dups_size":0,"ondisk_log_size":6309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:16.686185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479002624,"num_objects":7309,"num_object_clones":0,"num_object_copies":21927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7309,"num_write_kb":467776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":450,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199257+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:46:05.199257+0000","last_peered":"2024-08-17T17:46:05.199257+0000","last_clean":"2024-08-17T17:46:05.199257+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:46:05.199257+0000","last_undegraded":"2024-08-17T17:46:05.199257+0000","last_fullsized":"2024-08-17T17:46:05.199257+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1150,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199365+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:46:05.199365+0000","last_peered":"2024-08-17T17:46:05.199365+0000","last_clean":"2024-08-17T17:46:05.199365+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:46:05.199365+0000","last_undegraded":"2024-08-17T17:46:05.199365+0000","last_fullsized":"2024-08-17T17:46:05.199365+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":552,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.593311+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:46:16.593311+0000","last_peered":"2024-08-17T17:46:16.593311+0000","last_clean":"2024-08-17T17:46:16.593311+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:46:16.593311+0000","last_undegraded":"2024-08-17T17:46:16.593311+0000","last_fullsized":"2024-08-17T17:46:16.593311+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":90,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199491+0000","last_change":"2024-08-17T17:44:45.659123+0000","last_active":"2024-08-17T17:46:05.199491+0000","last_peered":"2024-08-17T17:46:05.199491+0000","last_clean":"2024-08-17T17:46:05.199491+0000","last_became_active":"2024-08-17T17:44:45.658785+0000","last_became_peered":"2024-08-17T17:44:45.658785+0000","last_unstale":"2024-08-17T17:46:05.199491+0000","last_undegraded":"2024-08-17T17:46:05.199491+0000","last_fullsized":"2024-08-17T17:46:05.199491+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:30.466001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"433'7335","reported_seq":7367,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.027708+0000","last_change":"2024-08-17T17:44:36.607785+0000","last_active":"2024-08-17T17:46:16.027708+0000","last_peered":"2024-08-17T17:46:16.027708+0000","last_clean":"2024-08-17T17:46:16.027708+0000","last_became_active":"2024-08-17T17:44:36.607472+0000","last_became_peered":"2024-08-17T17:44:36.607472+0000","last_unstale":"2024-08-17T17:46:16.027708+0000","last_undegraded":"2024-08-17T17:46:16.027708+0000","last_fullsized":"2024-08-17T17:46:16.027708+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:39.734707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480706560,"num_objects":7335,"num_object_clones":0,"num_object_copies":22005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7335,"num_write_kb":469440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":744,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214930+0000","last_change":"2024-08-17T17:44:52.091382+0000","last_active":"2024-08-17T17:46:05.214930+0000","last_peered":"2024-08-17T17:46:05.214930+0000","last_clean":"2024-08-17T17:46:05.214930+0000","last_became_active":"2024-08-17T17:44:48.663244+0000","last_became_peered":"2024-08-17T17:44:48.663244+0000","last_unstale":"2024-08-17T17:46:05.214930+0000","last_undegraded":"2024-08-17T17:46:05.214930+0000","last_fullsized":"2024-08-17T17:46:05.214930+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:44:52.091249+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:18.054213+0000","scrub_duration":0.001820239,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199684+0000","last_change":"2024-08-17T17:44:45.653860+0000","last_active":"2024-08-17T17:46:05.199684+0000","last_peered":"2024-08-17T17:46:05.199684+0000","last_clean":"2024-08-17T17:46:05.199684+0000","last_became_active":"2024-08-17T17:44:45.652151+0000","last_became_peered":"2024-08-17T17:44:45.652151+0000","last_unstale":"2024-08-17T17:46:05.199684+0000","last_undegraded":"2024-08-17T17:46:05.199684+0000","last_fullsized":"2024-08-17T17:46:05.199684+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:17.052902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":1019,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603663+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:46:16.603663+0000","last_peered":"2024-08-17T17:46:16.603663+0000","last_clean":"2024-08-17T17:46:16.603663+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:46:16.603663+0000","last_undegraded":"2024-08-17T17:46:16.603663+0000","last_fullsized":"2024-08-17T17:46:16.603663+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":743,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.605214+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:46:16.605214+0000","last_peered":"2024-08-17T17:46:16.605214+0000","last_clean":"2024-08-17T17:46:16.605214+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:46:16.605214+0000","last_undegraded":"2024-08-17T17:46:16.605214+0000","last_fullsized":"2024-08-17T17:46:16.605214+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"433'7268","reported_seq":7300,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.017454+0000","last_change":"2024-08-17T17:44:36.611239+0000","last_active":"2024-08-17T17:46:16.017454+0000","last_peered":"2024-08-17T17:46:16.017454+0000","last_clean":"2024-08-17T17:46:16.017454+0000","last_became_active":"2024-08-17T17:44:36.610907+0000","last_became_peered":"2024-08-17T17:44:36.610907+0000","last_unstale":"2024-08-17T17:46:16.017454+0000","last_undegraded":"2024-08-17T17:46:16.017454+0000","last_fullsized":"2024-08-17T17:46:16.017454+0000","mapping_epoch":410,"log_start":"418'1100","ondisk_log_start":"418'1100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:43.474878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476315648,"num_objects":7268,"num_object_clones":0,"num_object_copies":21804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7268,"num_write_kb":465152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":544,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506333+0000","last_change":"2024-08-17T17:44:48.665077+0000","last_active":"2024-08-17T17:46:10.506333+0000","last_peered":"2024-08-17T17:46:10.506333+0000","last_clean":"2024-08-17T17:46:10.506333+0000","last_became_active":"2024-08-17T17:44:48.664818+0000","last_became_peered":"2024-08-17T17:44:48.664818+0000","last_unstale":"2024-08-17T17:46:10.506333+0000","last_undegraded":"2024-08-17T17:46:10.506333+0000","last_fullsized":"2024-08-17T17:46:10.506333+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:36.058763+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":740,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603572+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:46:16.603572+0000","last_peered":"2024-08-17T17:46:16.603572+0000","last_clean":"2024-08-17T17:46:16.603572+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:46:16.603572+0000","last_undegraded":"2024-08-17T17:46:16.603572+0000","last_fullsized":"2024-08-17T17:46:16.603572+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1003,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214978+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:46:05.214978+0000","last_peered":"2024-08-17T17:46:05.214978+0000","last_clean":"2024-08-17T17:46:05.214978+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:46:05.214978+0000","last_undegraded":"2024-08-17T17:46:05.214978+0000","last_fullsized":"2024-08-17T17:46:05.214978+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":735,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506957+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:46:10.506957+0000","last_peered":"2024-08-17T17:46:10.506957+0000","last_clean":"2024-08-17T17:46:10.506957+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:46:10.506957+0000","last_undegraded":"2024-08-17T17:46:10.506957+0000","last_fullsized":"2024-08-17T17:46:10.506957+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"433'7323","reported_seq":7355,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.985687+0000","last_change":"2024-08-17T17:44:36.966926+0000","last_active":"2024-08-17T17:46:15.985687+0000","last_peered":"2024-08-17T17:46:15.985687+0000","last_clean":"2024-08-17T17:46:15.985687+0000","last_became_active":"2024-08-17T17:44:36.966608+0000","last_became_peered":"2024-08-17T17:44:36.966608+0000","last_unstale":"2024-08-17T17:46:15.985687+0000","last_undegraded":"2024-08-17T17:46:15.985687+0000","last_fullsized":"2024-08-17T17:46:15.985687+0000","mapping_epoch":410,"log_start":"418'1200","ondisk_log_start":"418'1200","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:39.384622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479920128,"num_objects":7323,"num_object_clones":0,"num_object_copies":21969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7323,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":451,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209746+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:46:05.209746+0000","last_peered":"2024-08-17T17:46:05.209746+0000","last_clean":"2024-08-17T17:46:05.209746+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:46:05.209746+0000","last_undegraded":"2024-08-17T17:46:05.209746+0000","last_fullsized":"2024-08-17T17:46:05.209746+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":55,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.215072+0000","last_change":"2024-08-17T17:44:45.670035+0000","last_active":"2024-08-17T17:46:05.215072+0000","last_peered":"2024-08-17T17:46:05.215072+0000","last_clean":"2024-08-17T17:46:05.215072+0000","last_became_active":"2024-08-17T17:44:45.655566+0000","last_became_peered":"2024-08-17T17:44:45.655566+0000","last_unstale":"2024-08-17T17:46:05.215072+0000","last_undegraded":"2024-08-17T17:46:05.215072+0000","last_fullsized":"2024-08-17T17:46:05.215072+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:17:32.230235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1250,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.604103+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:46:16.604103+0000","last_peered":"2024-08-17T17:46:16.604103+0000","last_clean":"2024-08-17T17:46:16.604103+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:46:16.604103+0000","last_undegraded":"2024-08-17T17:46:16.604103+0000","last_fullsized":"2024-08-17T17:46:16.604103+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.213964+0000","last_change":"2024-08-17T17:44:45.656210+0000","last_active":"2024-08-17T17:46:05.213964+0000","last_peered":"2024-08-17T17:46:05.213964+0000","last_clean":"2024-08-17T17:46:05.213964+0000","last_became_active":"2024-08-17T17:44:45.655853+0000","last_became_peered":"2024-08-17T17:44:45.655853+0000","last_unstale":"2024-08-17T17:46:05.213964+0000","last_undegraded":"2024-08-17T17:46:05.213964+0000","last_fullsized":"2024-08-17T17:46:05.213964+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:01:57.057139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"433'7231","reported_seq":7272,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:14.821168+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:14.821168+0000","last_peered":"2024-08-17T17:46:14.821168+0000","last_clean":"2024-08-17T17:46:14.821168+0000","last_became_active":"2024-08-17T17:44:45.658966+0000","last_became_peered":"2024-08-17T17:44:45.658966+0000","last_unstale":"2024-08-17T17:46:14.821168+0000","last_undegraded":"2024-08-17T17:46:14.821168+0000","last_fullsized":"2024-08-17T17:46:14.821168+0000","mapping_epoch":413,"log_start":"418'900","ondisk_log_start":"418'900","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:22.822524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473890816,"num_objects":7231,"num_object_clones":0,"num_object_copies":21693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7231,"num_write_kb":462784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":419,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199577+0000","last_change":"2024-08-17T17:44:51.112004+0000","last_active":"2024-08-17T17:46:05.199577+0000","last_peered":"2024-08-17T17:46:05.199577+0000","last_clean":"2024-08-17T17:46:05.199577+0000","last_became_active":"2024-08-17T17:44:48.661437+0000","last_became_peered":"2024-08-17T17:44:48.661437+0000","last_unstale":"2024-08-17T17:46:05.199577+0000","last_undegraded":"2024-08-17T17:46:05.199577+0000","last_fullsized":"2024-08-17T17:46:05.199577+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:00.448235+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":747,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506400+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:46:10.506400+0000","last_peered":"2024-08-17T17:46:10.506400+0000","last_clean":"2024-08-17T17:46:10.506400+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:46:10.506400+0000","last_undegraded":"2024-08-17T17:46:10.506400+0000","last_fullsized":"2024-08-17T17:46:10.506400+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":67,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214852+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:46:05.214852+0000","last_peered":"2024-08-17T17:46:05.214852+0000","last_clean":"2024-08-17T17:46:05.214852+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:46:05.214852+0000","last_undegraded":"2024-08-17T17:46:05.214852+0000","last_fullsized":"2024-08-17T17:46:05.214852+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209470+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:46:05.209470+0000","last_peered":"2024-08-17T17:46:05.209470+0000","last_clean":"2024-08-17T17:46:05.209470+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:46:05.209470+0000","last_undegraded":"2024-08-17T17:46:05.209470+0000","last_fullsized":"2024-08-17T17:46:05.209470+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":341,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.213029+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:46:05.213029+0000","last_peered":"2024-08-17T17:46:05.213029+0000","last_clean":"2024-08-17T17:46:05.213029+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:46:05.213029+0000","last_undegraded":"2024-08-17T17:46:05.213029+0000","last_fullsized":"2024-08-17T17:46:05.213029+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":351,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.210055+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:46:05.210055+0000","last_peered":"2024-08-17T17:46:05.210055+0000","last_clean":"2024-08-17T17:46:05.210055+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:46:05.210055+0000","last_undegraded":"2024-08-17T17:46:05.210055+0000","last_fullsized":"2024-08-17T17:46:05.210055+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":725,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.605083+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:46:16.605083+0000","last_peered":"2024-08-17T17:46:16.605083+0000","last_clean":"2024-08-17T17:46:16.605083+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:46:16.605083+0000","last_undegraded":"2024-08-17T17:46:16.605083+0000","last_fullsized":"2024-08-17T17:46:16.605083+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":450,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214891+0000","last_change":"2024-08-17T17:44:45.668086+0000","last_active":"2024-08-17T17:46:05.214891+0000","last_peered":"2024-08-17T17:46:05.214891+0000","last_clean":"2024-08-17T17:46:05.214891+0000","last_became_active":"2024-08-17T17:44:45.654205+0000","last_became_peered":"2024-08-17T17:44:45.654205+0000","last_unstale":"2024-08-17T17:46:05.214891+0000","last_undegraded":"2024-08-17T17:46:05.214891+0000","last_fullsized":"2024-08-17T17:46:05.214891+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:30.323982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199617+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:46:05.199617+0000","last_peered":"2024-08-17T17:46:05.199617+0000","last_clean":"2024-08-17T17:46:05.199617+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:46:05.199617+0000","last_undegraded":"2024-08-17T17:46:05.199617+0000","last_fullsized":"2024-08-17T17:46:05.199617+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":335,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.214539+0000","last_change":"2024-08-17T17:44:45.652198+0000","last_active":"2024-08-17T17:46:05.214539+0000","last_peered":"2024-08-17T17:46:05.214539+0000","last_clean":"2024-08-17T17:46:05.214539+0000","last_became_active":"2024-08-17T17:44:45.651854+0000","last_became_peered":"2024-08-17T17:44:45.651854+0000","last_unstale":"2024-08-17T17:46:05.214539+0000","last_undegraded":"2024-08-17T17:46:05.214539+0000","last_fullsized":"2024-08-17T17:46:05.214539+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:49:58.681636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":755,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506122+0000","last_change":"2024-08-17T17:44:45.651808+0000","last_active":"2024-08-17T17:46:10.506122+0000","last_peered":"2024-08-17T17:46:10.506122+0000","last_clean":"2024-08-17T17:46:10.506122+0000","last_became_active":"2024-08-17T17:44:45.651478+0000","last_became_peered":"2024-08-17T17:44:45.651478+0000","last_unstale":"2024-08-17T17:46:10.506122+0000","last_undegraded":"2024-08-17T17:46:10.506122+0000","last_fullsized":"2024-08-17T17:46:10.506122+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:04.589247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":406,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603030+0000","last_change":"2024-08-17T17:44:45.661728+0000","last_active":"2024-08-17T17:46:16.603030+0000","last_peered":"2024-08-17T17:46:16.603030+0000","last_clean":"2024-08-17T17:46:16.603030+0000","last_became_active":"2024-08-17T17:44:45.661337+0000","last_became_peered":"2024-08-17T17:44:45.661337+0000","last_unstale":"2024-08-17T17:46:16.603030+0000","last_undegraded":"2024-08-17T17:46:16.603030+0000","last_fullsized":"2024-08-17T17:46:16.603030+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:54:21.723874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":229,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.199868+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:46:05.199868+0000","last_peered":"2024-08-17T17:46:05.199868+0000","last_clean":"2024-08-17T17:46:05.199868+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:46:05.199868+0000","last_undegraded":"2024-08-17T17:46:05.199868+0000","last_fullsized":"2024-08-17T17:46:05.199868+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507401+0000","last_change":"2024-08-17T17:44:45.663565+0000","last_active":"2024-08-17T17:46:10.507401+0000","last_peered":"2024-08-17T17:46:10.507401+0000","last_clean":"2024-08-17T17:46:10.507401+0000","last_became_active":"2024-08-17T17:44:45.663237+0000","last_became_peered":"2024-08-17T17:44:45.663237+0000","last_unstale":"2024-08-17T17:46:10.507401+0000","last_undegraded":"2024-08-17T17:46:10.507401+0000","last_fullsized":"2024-08-17T17:46:10.507401+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:26:34.711189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":729,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506497+0000","last_change":"2024-08-17T17:44:45.662891+0000","last_active":"2024-08-17T17:46:10.506497+0000","last_peered":"2024-08-17T17:46:10.506497+0000","last_clean":"2024-08-17T17:46:10.506497+0000","last_became_active":"2024-08-17T17:44:45.662580+0000","last_became_peered":"2024-08-17T17:44:45.662580+0000","last_unstale":"2024-08-17T17:46:10.506497+0000","last_undegraded":"2024-08-17T17:46:10.506497+0000","last_fullsized":"2024-08-17T17:46:10.506497+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:48.627337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":734,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603155+0000","last_change":"2024-08-17T17:44:45.657935+0000","last_active":"2024-08-17T17:46:16.603155+0000","last_peered":"2024-08-17T17:46:16.603155+0000","last_clean":"2024-08-17T17:46:16.603155+0000","last_became_active":"2024-08-17T17:44:45.657525+0000","last_became_peered":"2024-08-17T17:44:45.657525+0000","last_unstale":"2024-08-17T17:46:16.603155+0000","last_undegraded":"2024-08-17T17:46:16.603155+0000","last_fullsized":"2024-08-17T17:46:16.603155+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:55.691041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603510+0000","last_change":"2024-08-17T17:44:45.661307+0000","last_active":"2024-08-17T17:46:16.603510+0000","last_peered":"2024-08-17T17:46:16.603510+0000","last_clean":"2024-08-17T17:46:16.603510+0000","last_became_active":"2024-08-17T17:44:45.660976+0000","last_became_peered":"2024-08-17T17:44:45.660976+0000","last_unstale":"2024-08-17T17:46:16.603510+0000","last_undegraded":"2024-08-17T17:46:16.603510+0000","last_fullsized":"2024-08-17T17:46:16.603510+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:22.725842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":325,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209596+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:46:05.209596+0000","last_peered":"2024-08-17T17:46:05.209596+0000","last_clean":"2024-08-17T17:46:05.209596+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:46:05.209596+0000","last_undegraded":"2024-08-17T17:46:05.209596+0000","last_fullsized":"2024-08-17T17:46:05.209596+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":262,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603853+0000","last_change":"2024-08-17T17:44:45.660322+0000","last_active":"2024-08-17T17:46:16.603853+0000","last_peered":"2024-08-17T17:46:16.603853+0000","last_clean":"2024-08-17T17:46:16.603853+0000","last_became_active":"2024-08-17T17:44:45.660018+0000","last_became_peered":"2024-08-17T17:44:45.660018+0000","last_unstale":"2024-08-17T17:46:16.603853+0000","last_undegraded":"2024-08-17T17:46:16.603853+0000","last_fullsized":"2024-08-17T17:46:16.603853+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:57.732705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":116,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506583+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:46:10.506583+0000","last_peered":"2024-08-17T17:46:10.506583+0000","last_clean":"2024-08-17T17:46:10.506583+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:46:10.506583+0000","last_undegraded":"2024-08-17T17:46:10.506583+0000","last_fullsized":"2024-08-17T17:46:10.506583+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":334,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.212966+0000","last_change":"2024-08-17T17:44:45.657401+0000","last_active":"2024-08-17T17:46:05.212966+0000","last_peered":"2024-08-17T17:46:05.212966+0000","last_clean":"2024-08-17T17:46:05.212966+0000","last_became_active":"2024-08-17T17:44:45.657074+0000","last_became_peered":"2024-08-17T17:44:45.657074+0000","last_unstale":"2024-08-17T17:46:05.212966+0000","last_undegraded":"2024-08-17T17:46:05.212966+0000","last_fullsized":"2024-08-17T17:46:05.212966+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:19.924661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":222,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500886+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:46:10.500886+0000","last_peered":"2024-08-17T17:46:10.500886+0000","last_clean":"2024-08-17T17:46:10.500886+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:46:10.500886+0000","last_undegraded":"2024-08-17T17:46:10.500886+0000","last_fullsized":"2024-08-17T17:46:10.500886+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":334,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209564+0000","last_change":"2024-08-17T17:44:45.657702+0000","last_active":"2024-08-17T17:46:05.209564+0000","last_peered":"2024-08-17T17:46:05.209564+0000","last_clean":"2024-08-17T17:46:05.209564+0000","last_became_active":"2024-08-17T17:44:45.657367+0000","last_became_peered":"2024-08-17T17:44:45.657367+0000","last_unstale":"2024-08-17T17:46:05.209564+0000","last_undegraded":"2024-08-17T17:46:05.209564+0000","last_fullsized":"2024-08-17T17:46:05.209564+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:29.447263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":232,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500732+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:46:10.500732+0000","last_peered":"2024-08-17T17:46:10.500732+0000","last_clean":"2024-08-17T17:46:10.500732+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:46:10.500732+0000","last_undegraded":"2024-08-17T17:46:10.500732+0000","last_fullsized":"2024-08-17T17:46:10.500732+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":131,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507923+0000","last_change":"2024-08-17T17:44:45.660616+0000","last_active":"2024-08-17T17:46:10.507923+0000","last_peered":"2024-08-17T17:46:10.507923+0000","last_clean":"2024-08-17T17:46:10.507923+0000","last_became_active":"2024-08-17T17:44:45.660198+0000","last_became_peered":"2024-08-17T17:44:45.660198+0000","last_unstale":"2024-08-17T17:46:10.507923+0000","last_undegraded":"2024-08-17T17:46:10.507923+0000","last_fullsized":"2024-08-17T17:46:10.507923+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:06.005702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":52,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603711+0000","last_change":"2024-08-17T17:44:45.662070+0000","last_active":"2024-08-17T17:46:16.603711+0000","last_peered":"2024-08-17T17:46:16.603711+0000","last_clean":"2024-08-17T17:46:16.603711+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:16.603711+0000","last_undegraded":"2024-08-17T17:46:16.603711+0000","last_fullsized":"2024-08-17T17:46:16.603711+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:11.915663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":340,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-17T17:46:05.209197+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:46:05.209197+0000","last_peered":"2024-08-17T17:46:05.209197+0000","last_clean":"2024-08-17T17:46:05.209197+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:46:05.209197+0000","last_undegraded":"2024-08-17T17:46:05.209197+0000","last_fullsized":"2024-08-17T17:46:05.209197+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":766,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506014+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:46:10.506014+0000","last_peered":"2024-08-17T17:46:10.506014+0000","last_clean":"2024-08-17T17:46:10.506014+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:46:10.506014+0000","last_undegraded":"2024-08-17T17:46:10.506014+0000","last_fullsized":"2024-08-17T17:46:10.506014+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":7627931648,"num_objects":116393,"num_object_clones":0,"num_object_copies":349179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":116393,"num_write_kb":7449152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22916890624,"data_stored":22916890624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109293,"ondisk_log_size":109293,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201793,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3243764,"kb_used_data":2324204,"kb_used_omap":1,"kb_used_meta":919550,"kb_avail":90501388,"statfs":{"total":95995035648,"available":92673421312,"internally_reserved":0,"allocated":2379984896,"data_stored":2377874804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":941619659},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922289,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3708264,"kb_used_data":3241632,"kb_used_omap":7,"kb_used_meta":466616,"kb_avail":90036888,"statfs":{"total":95995035648,"available":92197773312,"internally_reserved":0,"allocated":3319431168,"data_stored":3317332874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":477815478},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085813,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2890216,"kb_used_data":2315872,"kb_used_omap":7,"kb_used_meta":574328,"kb_avail":90854936,"statfs":{"total":95995035648,"available":93035454464,"internally_reserved":0,"allocated":2371452928,"data_stored":2369351367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":588112313},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986703,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5529200,"kb_used_data":4649832,"kb_used_omap":1,"kb_used_meta":879358,"kb_avail":88215952,"statfs":{"total":95995035648,"available":90333134848,"internally_reserved":0,"allocated":4761427968,"data_stored":4759321910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":900463046},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,2,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":468151435594,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599121,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4221536,"kb_used_data":3269400,"kb_used_omap":7,"kb_used_meta":952120,"kb_avail":89523616,"statfs":{"total":95995035648,"available":91672182784,"internally_reserved":0,"allocated":3347865600,"data_stored":3345771019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":974971053},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762647,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2992148,"kb_used_data":2152460,"kb_used_omap":7,"kb_used_meta":839672,"kb_avail":90753004,"statfs":{"total":95995035648,"available":92931076096,"internally_reserved":0,"allocated":2204119040,"data_stored":2202035333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":859824545},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926173,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5621416,"kb_used_data":4651232,"kb_used_omap":7,"kb_used_meta":970168,"kb_avail":88123736,"statfs":{"total":95995035648,"available":90238705664,"internally_reserved":0,"allocated":4762861568,"data_stored":4760759779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":993452636},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4786683904,"data_stored":4786683904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1917845504,"data_stored":1917845504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3347709952,"data_stored":3347709952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4778688512,"data_stored":4778688512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2381840384,"data_stored":2381840384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3318480896,"data_stored":3318480896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2385641472,"data_stored":2385641472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:46:20.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:46:20.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:20.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:20.816 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 121018 121002 87.922 60.5 0.0191716 0.0113684 2024-08-17T17:46:20.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:20 smithi064 ceph-mon[104516]: pgmap v1394: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-08-17T17:46:20.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:20 smithi064 ceph-mon[109246]: pgmap v1394: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-08-17T17:46:20.834 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:20.872 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:20 smithi175 ceph-mon[101434]: pgmap v1394: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-08-17T17:46:20.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:21.027 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:21.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:21.167 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:21.269 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:21.298 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:21.323 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:46:21.324 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:21.324 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:21.333 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:21.354 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:21.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:46:21.363 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:21.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:21.402 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:46:21.402 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:21.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:21.435 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:46:21.435 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:21.481 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:46:21.481 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:21.517 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:46:21.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:21.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:21.555 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:46:21.555 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:21.595 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:46:21.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:21.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1885105398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:21.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1885105398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:21.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:21.702 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:21.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:21.816 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 121993 121977 87.6112 60.9375 0.00815918 0.0114091 2024-08-17T17:46:21.836 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1885105398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:21.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.978 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:21.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:22.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:22.110 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:22.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.245 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:22.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:22.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:22.383 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:22.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:22.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:22.627 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:22.664 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:22.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.805 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:22.816 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 122983 122967 87.3187 61.875 0.00971286 0.0114473 2024-08-17T17:46:22.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:22 smithi064 ceph-mon[104516]: pgmap v1395: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:22.900 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:22 smithi064 ceph-mon[109246]: pgmap v1395: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:22.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:22 smithi175 ceph-mon[101434]: pgmap v1395: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:46:22.945 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:23.047 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:23.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:23.184 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:23.224 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:23.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:23.365 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:23.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:23.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:23.606 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:23.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:23.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:23.777 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:23.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 123978 123962 87.0363 62.1875 0.0125312 0.0114851 2024-08-17T17:46:23.879 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.919 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:23.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:23.999 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:23] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.51.0" 2024-08-17T17:46:24.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.059 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:24.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:24.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:24.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:24.294 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:24.329 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:24.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:24.467 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:24.569 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:24.604 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:24.706 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:24.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:24.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:24 smithi064 ceph-mon[104516]: pgmap v1396: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:46:24.781 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:24 smithi064 ceph-mon[109246]: pgmap v1396: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0106 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 124978 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 86.7801 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 41.1352 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 244.312 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 38.125 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1388 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 658.163 2024-08-17T17:46:24.817 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3909 2024-08-17T17:46:24.818 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 610 2024-08-17T17:46:24.818 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.011519 2024-08-17T17:46:24.818 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0127198 2024-08-17T17:46:24.818 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.515984 2024-08-17T17:46:24.818 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00139059 2024-08-17T17:46:24.821 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:46:24.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:24.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:24 smithi175 ceph-mon[101434]: pgmap v1396: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:46:24.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.028 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:25.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:25.129 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:25.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.172 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:25.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:25.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:25.303 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:25.404 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:25.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:25.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:25.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:25.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:25.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:25.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:25.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:25.811 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:25.851 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:25.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:25.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:25.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:25.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:26.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:26.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:26.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:26.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:26.261 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:26.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:26.400 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:26.502 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:26.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:26.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:26.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:26.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:26 smithi064 ceph-mon[104516]: pgmap v1397: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2024-08-17T17:46:26.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:26 smithi064 ceph-mon[109246]: pgmap v1397: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2024-08-17T17:46:26.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:26.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:26.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:26 smithi175 ceph-mon[101434]: pgmap v1397: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2024-08-17T17:46:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:26.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:27.060 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:27.100 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:27.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:27.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:27.236 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:27.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:27.377 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:27.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:27.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:27.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:27.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:27.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:27.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:27.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:27.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:27.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:27.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:28.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:28.077 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:28.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:28.210 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:28.250 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:46:28.252 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:46:28.312 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:28.346 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:28.448 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:28.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:46:28.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:28.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.631 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:28.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:28.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:28 smithi064 ceph-mon[104516]: pgmap v1398: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 94 B/s rd, 53 MiB/s wr, 950 op/s 2024-08-17T17:46:28.727 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:28 smithi064 ceph-mon[109246]: pgmap v1398: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 94 B/s rd, 53 MiB/s wr, 950 op/s 2024-08-17T17:46:28.732 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:28.765 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:28.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:28.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:28.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:28 smithi175 ceph-mon[101434]: pgmap v1398: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 94 B/s rd, 53 MiB/s wr, 950 op/s 2024-08-17T17:46:29.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:29.041 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:29.143 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:29.184 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:29.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:29.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:29.316 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:29.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:29.463 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1398,"stamp":"2024-08-17T17:46:27.372330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8122341373,"num_objects":123893,"num_object_clones":0,"num_object_copies":371679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":123893,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":127001,"num_write_kb":8011218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8657757,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114665,"ondisk_log_size":114665,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":29842476,"kb_used_data":24191860,"kb_used_omap":40,"kb_used_meta":5650391,"kb_avail":626373588,"statfs":{"total":671965249536,"available":641406554112,"internally_reserved":0,"allocated":24772464640,"data_stored":24757732827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41814,"internal_metadata":5786000554},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":529399832,"num_objects":8079,"num_object_clones":0,"num_object_copies":24237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8079,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10307,"num_write_kb":588289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.841063"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603751+0000","last_change":"2024-08-17T17:44:45.659658+0000","last_active":"2024-08-17T17:46:16.603751+0000","last_peered":"2024-08-17T17:46:16.603751+0000","last_clean":"2024-08-17T17:46:16.603751+0000","last_became_active":"2024-08-17T17:44:45.659189+0000","last_became_peered":"2024-08-17T17:44:45.659189+0000","last_unstale":"2024-08-17T17:46:16.603751+0000","last_undegraded":"2024-08-17T17:46:16.603751+0000","last_fullsized":"2024-08-17T17:46:16.603751+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:45.422590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":436,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.503641+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:46:10.503641+0000","last_peered":"2024-08-17T17:46:10.503641+0000","last_clean":"2024-08-17T17:46:10.503641+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:46:10.503641+0000","last_undegraded":"2024-08-17T17:46:10.503641+0000","last_fullsized":"2024-08-17T17:46:10.503641+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500795+0000","last_change":"2024-08-17T17:44:45.650591+0000","last_active":"2024-08-17T17:46:10.500795+0000","last_peered":"2024-08-17T17:46:10.500795+0000","last_clean":"2024-08-17T17:46:10.500795+0000","last_became_active":"2024-08-17T17:44:45.650249+0000","last_became_peered":"2024-08-17T17:44:45.650249+0000","last_unstale":"2024-08-17T17:46:10.500795+0000","last_undegraded":"2024-08-17T17:46:10.500795+0000","last_fullsized":"2024-08-17T17:46:10.500795+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:06.858698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":729,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594849+0000","last_change":"2024-08-17T17:44:45.663771+0000","last_active":"2024-08-17T17:46:16.594849+0000","last_peered":"2024-08-17T17:46:16.594849+0000","last_clean":"2024-08-17T17:46:16.594849+0000","last_became_active":"2024-08-17T17:44:45.663321+0000","last_became_peered":"2024-08-17T17:44:45.663321+0000","last_unstale":"2024-08-17T17:46:16.594849+0000","last_undegraded":"2024-08-17T17:46:16.594849+0000","last_fullsized":"2024-08-17T17:46:16.594849+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:30.374220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":729,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599781+0000","last_change":"2024-08-17T17:44:45.657665+0000","last_active":"2024-08-17T17:46:16.599781+0000","last_peered":"2024-08-17T17:46:16.599781+0000","last_clean":"2024-08-17T17:46:16.599781+0000","last_became_active":"2024-08-17T17:44:45.657342+0000","last_became_peered":"2024-08-17T17:44:45.657342+0000","last_unstale":"2024-08-17T17:46:16.599781+0000","last_undegraded":"2024-08-17T17:46:16.599781+0000","last_fullsized":"2024-08-17T17:46:16.599781+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:00:18.245759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":422,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504924+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:46:10.504924+0000","last_peered":"2024-08-17T17:46:10.504924+0000","last_clean":"2024-08-17T17:46:10.504924+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:46:10.504924+0000","last_undegraded":"2024-08-17T17:46:10.504924+0000","last_fullsized":"2024-08-17T17:46:10.504924+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":94,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504466+0000","last_change":"2024-08-17T17:44:45.660525+0000","last_active":"2024-08-17T17:46:10.504466+0000","last_peered":"2024-08-17T17:46:10.504466+0000","last_clean":"2024-08-17T17:46:10.504466+0000","last_became_active":"2024-08-17T17:44:45.660152+0000","last_became_peered":"2024-08-17T17:44:45.660152+0000","last_unstale":"2024-08-17T17:46:10.504466+0000","last_undegraded":"2024-08-17T17:46:10.504466+0000","last_fullsized":"2024-08-17T17:46:10.504466+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:35:09.946483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":370,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.513561+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:46:10.513561+0000","last_peered":"2024-08-17T17:46:10.513561+0000","last_clean":"2024-08-17T17:46:10.513561+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:46:10.513561+0000","last_undegraded":"2024-08-17T17:46:10.513561+0000","last_fullsized":"2024-08-17T17:46:10.513561+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"434'7911","reported_seq":7944,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.582200+0000","last_change":"2024-08-17T17:44:36.608512+0000","last_active":"2024-08-17T17:46:25.582200+0000","last_peered":"2024-08-17T17:46:25.582200+0000","last_clean":"2024-08-17T17:46:25.582200+0000","last_became_active":"2024-08-17T17:44:36.608224+0000","last_became_peered":"2024-08-17T17:44:36.608224+0000","last_unstale":"2024-08-17T17:46:25.582200+0000","last_undegraded":"2024-08-17T17:46:25.582200+0000","last_fullsized":"2024-08-17T17:46:25.582200+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7911,"log_dups_size":0,"ondisk_log_size":7911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:46.388975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":512425984,"num_objects":7819,"num_object_clones":0,"num_object_copies":23457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7911,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":432,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534339+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:46:16.534339+0000","last_peered":"2024-08-17T17:46:16.534339+0000","last_clean":"2024-08-17T17:46:16.534339+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:46:16.534339+0000","last_undegraded":"2024-08-17T17:46:16.534339+0000","last_fullsized":"2024-08-17T17:46:16.534339+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533208+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:46:16.533208+0000","last_peered":"2024-08-17T17:46:16.533208+0000","last_clean":"2024-08-17T17:46:16.533208+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:46:16.533208+0000","last_undegraded":"2024-08-17T17:46:16.533208+0000","last_fullsized":"2024-08-17T17:46:16.533208+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"434'7754","reported_seq":7786,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.600761+0000","last_change":"2024-08-17T17:44:45.652247+0000","last_active":"2024-08-17T17:46:25.600761+0000","last_peered":"2024-08-17T17:46:25.600761+0000","last_clean":"2024-08-17T17:46:25.600761+0000","last_became_active":"2024-08-17T17:44:45.651926+0000","last_became_peered":"2024-08-17T17:44:45.651926+0000","last_unstale":"2024-08-17T17:46:25.600761+0000","last_undegraded":"2024-08-17T17:46:25.600761+0000","last_fullsized":"2024-08-17T17:46:25.600761+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:39.517297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":500563968,"num_objects":7638,"num_object_clones":0,"num_object_copies":22914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7754,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":433,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504157+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:46:10.504157+0000","last_peered":"2024-08-17T17:46:10.504157+0000","last_clean":"2024-08-17T17:46:10.504157+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:46:10.504157+0000","last_undegraded":"2024-08-17T17:46:10.504157+0000","last_fullsized":"2024-08-17T17:46:10.504157+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":337,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.027997+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:16.027997+0000","last_peered":"2024-08-17T17:46:16.027997+0000","last_clean":"2024-08-17T17:46:16.027997+0000","last_became_active":"2024-08-17T17:44:45.658981+0000","last_became_peered":"2024-08-17T17:44:45.658981+0000","last_unstale":"2024-08-17T17:46:16.027997+0000","last_undegraded":"2024-08-17T17:46:16.027997+0000","last_fullsized":"2024-08-17T17:46:16.027997+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:59.247763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"434'7908","reported_seq":7972,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.430892+0000","last_change":"2024-08-17T17:44:45.662022+0000","last_active":"2024-08-17T17:46:26.430892+0000","last_peered":"2024-08-17T17:46:26.430892+0000","last_clean":"2024-08-17T17:46:26.430892+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:26.430892+0000","last_undegraded":"2024-08-17T17:46:26.430892+0000","last_fullsized":"2024-08-17T17:46:26.430892+0000","mapping_epoch":413,"log_start":"418'500","ondisk_log_start":"418'500","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:02.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":503185408,"num_objects":7678,"num_object_clones":0,"num_object_copies":23034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7908,"num_write_kb":498752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":450,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.506023+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:46:10.506023+0000","last_peered":"2024-08-17T17:46:10.506023+0000","last_clean":"2024-08-17T17:46:10.506023+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:46:10.506023+0000","last_undegraded":"2024-08-17T17:46:10.506023+0000","last_fullsized":"2024-08-17T17:46:10.506023+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":350,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.514005+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:46:10.514005+0000","last_peered":"2024-08-17T17:46:10.514005+0000","last_clean":"2024-08-17T17:46:10.514005+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:46:10.514005+0000","last_undegraded":"2024-08-17T17:46:10.514005+0000","last_fullsized":"2024-08-17T17:46:10.514005+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"434'7940","reported_seq":8006,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.413640+0000","last_change":"2024-08-17T17:44:45.658418+0000","last_active":"2024-08-17T17:46:26.413640+0000","last_peered":"2024-08-17T17:46:26.413640+0000","last_clean":"2024-08-17T17:46:26.413640+0000","last_became_active":"2024-08-17T17:44:45.658105+0000","last_became_peered":"2024-08-17T17:44:45.658105+0000","last_unstale":"2024-08-17T17:46:26.413640+0000","last_undegraded":"2024-08-17T17:46:26.413640+0000","last_fullsized":"2024-08-17T17:46:26.413640+0000","mapping_epoch":413,"log_start":"418'600","ondisk_log_start":"418'600","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7340,"log_dups_size":0,"ondisk_log_size":7340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:47.860621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505544704,"num_objects":7714,"num_object_clones":0,"num_object_copies":23142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7940,"num_write_kb":500928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":238,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500997+0000","last_change":"2024-08-17T17:44:45.657383+0000","last_active":"2024-08-17T17:46:10.500997+0000","last_peered":"2024-08-17T17:46:10.500997+0000","last_clean":"2024-08-17T17:46:10.500997+0000","last_became_active":"2024-08-17T17:44:45.656436+0000","last_became_peered":"2024-08-17T17:44:45.656436+0000","last_unstale":"2024-08-17T17:46:10.500997+0000","last_undegraded":"2024-08-17T17:46:10.500997+0000","last_fullsized":"2024-08-17T17:46:10.500997+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:23:40.172334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":743,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599565+0000","last_change":"2024-08-17T17:45:27.582532+0000","last_active":"2024-08-17T17:46:16.599565+0000","last_peered":"2024-08-17T17:46:16.599565+0000","last_clean":"2024-08-17T17:46:16.599565+0000","last_became_active":"2024-08-17T17:45:27.582184+0000","last_became_peered":"2024-08-17T17:45:27.582184+0000","last_unstale":"2024-08-17T17:46:16.599565+0000","last_undegraded":"2024-08-17T17:46:16.599565+0000","last_fullsized":"2024-08-17T17:46:16.599565+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:28.064920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"434'7945","reported_seq":8004,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.626212+0000","last_change":"2024-08-17T17:44:36.607826+0000","last_active":"2024-08-17T17:46:26.626212+0000","last_peered":"2024-08-17T17:46:26.626212+0000","last_clean":"2024-08-17T17:46:26.626212+0000","last_became_active":"2024-08-17T17:44:36.607543+0000","last_became_peered":"2024-08-17T17:44:36.607543+0000","last_unstale":"2024-08-17T17:46:26.626212+0000","last_undegraded":"2024-08-17T17:46:26.626212+0000","last_fullsized":"2024-08-17T17:46:26.626212+0000","mapping_epoch":410,"log_start":"418'1100","ondisk_log_start":"418'1100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:22.366195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504430592,"num_objects":7697,"num_object_clones":0,"num_object_copies":23091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7945,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":808,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504775+0000","last_change":"2024-08-17T17:44:48.661664+0000","last_active":"2024-08-17T17:46:10.504775+0000","last_peered":"2024-08-17T17:46:10.504775+0000","last_clean":"2024-08-17T17:46:10.504775+0000","last_became_active":"2024-08-17T17:44:48.661277+0000","last_became_peered":"2024-08-17T17:44:48.661277+0000","last_unstale":"2024-08-17T17:46:10.504775+0000","last_undegraded":"2024-08-17T17:46:10.504775+0000","last_fullsized":"2024-08-17T17:46:10.504775+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.305103+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":79,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.595018+0000","last_change":"2024-08-17T17:44:45.670332+0000","last_active":"2024-08-17T17:46:16.595018+0000","last_peered":"2024-08-17T17:46:16.595018+0000","last_clean":"2024-08-17T17:46:16.595018+0000","last_became_active":"2024-08-17T17:44:45.660508+0000","last_became_peered":"2024-08-17T17:44:45.660508+0000","last_unstale":"2024-08-17T17:46:16.595018+0000","last_undegraded":"2024-08-17T17:46:16.595018+0000","last_fullsized":"2024-08-17T17:46:16.595018+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:17.489964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":360,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.513269+0000","last_change":"2024-08-17T17:44:45.656521+0000","last_active":"2024-08-17T17:46:10.513269+0000","last_peered":"2024-08-17T17:46:10.513269+0000","last_clean":"2024-08-17T17:46:10.513269+0000","last_became_active":"2024-08-17T17:44:45.656178+0000","last_became_peered":"2024-08-17T17:44:45.656178+0000","last_unstale":"2024-08-17T17:46:10.513269+0000","last_undegraded":"2024-08-17T17:46:10.513269+0000","last_fullsized":"2024-08-17T17:46:10.513269+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:48.486263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"434'7834","reported_seq":7876,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.562077+0000","last_change":"2024-08-17T17:44:45.653325+0000","last_active":"2024-08-17T17:46:25.562077+0000","last_peered":"2024-08-17T17:46:25.562077+0000","last_clean":"2024-08-17T17:46:25.562077+0000","last_became_active":"2024-08-17T17:44:45.652943+0000","last_became_peered":"2024-08-17T17:44:45.652943+0000","last_unstale":"2024-08-17T17:46:25.562077+0000","last_undegraded":"2024-08-17T17:46:25.562077+0000","last_fullsized":"2024-08-17T17:46:25.562077+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7834,"log_dups_size":0,"ondisk_log_size":7834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:29:49.759034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":507641856,"num_objects":7746,"num_object_clones":0,"num_object_copies":23238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7834,"num_write_kb":498560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":472,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:11.526491+0000","last_change":"2024-08-17T17:44:48.661693+0000","last_active":"2024-08-17T17:46:11.526491+0000","last_peered":"2024-08-17T17:46:11.526491+0000","last_clean":"2024-08-17T17:46:11.526491+0000","last_became_active":"2024-08-17T17:44:48.661280+0000","last_became_peered":"2024-08-17T17:44:48.661280+0000","last_unstale":"2024-08-17T17:46:11.526491+0000","last_undegraded":"2024-08-17T17:46:11.526491+0000","last_fullsized":"2024-08-17T17:46:11.526491+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:56.378196+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":357,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.509171+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:46:10.509171+0000","last_peered":"2024-08-17T17:46:10.509171+0000","last_clean":"2024-08-17T17:46:10.509171+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:46:10.509171+0000","last_undegraded":"2024-08-17T17:46:10.509171+0000","last_fullsized":"2024-08-17T17:46:10.509171+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":327,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507781+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:46:10.507781+0000","last_peered":"2024-08-17T17:46:10.507781+0000","last_clean":"2024-08-17T17:46:10.507781+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:46:10.507781+0000","last_undegraded":"2024-08-17T17:46:10.507781+0000","last_fullsized":"2024-08-17T17:46:10.507781+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"434'7875","reported_seq":7908,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:24.808930+0000","last_change":"2024-08-17T17:44:36.610056+0000","last_active":"2024-08-17T17:46:24.808930+0000","last_peered":"2024-08-17T17:46:24.808930+0000","last_clean":"2024-08-17T17:46:24.808930+0000","last_became_active":"2024-08-17T17:44:36.609357+0000","last_became_peered":"2024-08-17T17:44:36.609357+0000","last_unstale":"2024-08-17T17:46:24.808930+0000","last_undegraded":"2024-08-17T17:46:24.808930+0000","last_fullsized":"2024-08-17T17:46:24.808930+0000","mapping_epoch":410,"log_start":"420'1600","ondisk_log_start":"420'1600","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:01.344940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515964928,"num_objects":7873,"num_object_clones":0,"num_object_copies":23619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7875,"num_write_kb":503936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":328,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.027954+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:46:16.027954+0000","last_peered":"2024-08-17T17:46:16.027954+0000","last_clean":"2024-08-17T17:46:16.027954+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:46:16.027954+0000","last_undegraded":"2024-08-17T17:46:16.027954+0000","last_fullsized":"2024-08-17T17:46:16.027954+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":342,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.509927+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:46:10.509927+0000","last_peered":"2024-08-17T17:46:10.509927+0000","last_clean":"2024-08-17T17:46:10.509927+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:46:10.509927+0000","last_undegraded":"2024-08-17T17:46:10.509927+0000","last_fullsized":"2024-08-17T17:46:10.509927+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"434'7836","reported_seq":7868,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.056144+0000","last_change":"2024-08-17T17:44:36.610369+0000","last_active":"2024-08-17T17:46:26.056144+0000","last_peered":"2024-08-17T17:46:26.056144+0000","last_clean":"2024-08-17T17:46:26.056144+0000","last_became_active":"2024-08-17T17:44:36.610096+0000","last_became_peered":"2024-08-17T17:44:36.610096+0000","last_unstale":"2024-08-17T17:46:26.056144+0000","last_undegraded":"2024-08-17T17:46:26.056144+0000","last_fullsized":"2024-08-17T17:46:26.056144+0000","mapping_epoch":410,"log_start":"420'1700","ondisk_log_start":"420'1700","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:28.841902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":501350400,"num_objects":7650,"num_object_clones":0,"num_object_copies":22950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7836,"num_write_kb":495552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":451,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:11.526537+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:46:11.526537+0000","last_peered":"2024-08-17T17:46:11.526537+0000","last_clean":"2024-08-17T17:46:11.526537+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:46:11.526537+0000","last_undegraded":"2024-08-17T17:46:11.526537+0000","last_fullsized":"2024-08-17T17:46:11.526537+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":116,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603229+0000","last_change":"2024-08-17T17:44:45.657126+0000","last_active":"2024-08-17T17:46:16.603229+0000","last_peered":"2024-08-17T17:46:16.603229+0000","last_clean":"2024-08-17T17:46:16.603229+0000","last_became_active":"2024-08-17T17:44:45.656799+0000","last_became_peered":"2024-08-17T17:44:45.656799+0000","last_unstale":"2024-08-17T17:46:16.603229+0000","last_undegraded":"2024-08-17T17:46:16.603229+0000","last_fullsized":"2024-08-17T17:46:16.603229+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:06.941588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":727,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.600001+0000","last_change":"2024-08-17T17:44:45.659396+0000","last_active":"2024-08-17T17:46:16.600001+0000","last_peered":"2024-08-17T17:46:16.600001+0000","last_clean":"2024-08-17T17:46:16.600001+0000","last_became_active":"2024-08-17T17:44:45.659081+0000","last_became_peered":"2024-08-17T17:44:45.659081+0000","last_unstale":"2024-08-17T17:46:16.600001+0000","last_undegraded":"2024-08-17T17:46:16.600001+0000","last_fullsized":"2024-08-17T17:46:16.600001+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:14.819296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"434'7934","reported_seq":7993,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.618335+0000","last_change":"2024-08-17T17:44:36.610395+0000","last_active":"2024-08-17T17:46:26.618335+0000","last_peered":"2024-08-17T17:46:26.618335+0000","last_clean":"2024-08-17T17:46:26.618335+0000","last_became_active":"2024-08-17T17:44:36.610060+0000","last_became_peered":"2024-08-17T17:44:36.610060+0000","last_unstale":"2024-08-17T17:46:26.618335+0000","last_undegraded":"2024-08-17T17:46:26.618335+0000","last_fullsized":"2024-08-17T17:46:26.618335+0000","mapping_epoch":410,"log_start":"418'1100","ondisk_log_start":"418'1100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:23.272119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504627200,"num_objects":7700,"num_object_clones":0,"num_object_copies":23100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7934,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":775,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502498+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:46:10.502498+0000","last_peered":"2024-08-17T17:46:10.502498+0000","last_clean":"2024-08-17T17:46:10.502498+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:46:10.502498+0000","last_undegraded":"2024-08-17T17:46:10.502498+0000","last_fullsized":"2024-08-17T17:46:10.502498+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1028,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.501191+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:46:10.501191+0000","last_peered":"2024-08-17T17:46:10.501191+0000","last_clean":"2024-08-17T17:46:10.501191+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:46:10.501191+0000","last_undegraded":"2024-08-17T17:46:10.501191+0000","last_fullsized":"2024-08-17T17:46:10.501191+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":361,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502934+0000","last_change":"2024-08-17T17:44:45.651175+0000","last_active":"2024-08-17T17:46:10.502934+0000","last_peered":"2024-08-17T17:46:10.502934+0000","last_clean":"2024-08-17T17:46:10.502934+0000","last_became_active":"2024-08-17T17:44:45.650847+0000","last_became_peered":"2024-08-17T17:44:45.650847+0000","last_unstale":"2024-08-17T17:46:10.502934+0000","last_undegraded":"2024-08-17T17:46:10.502934+0000","last_fullsized":"2024-08-17T17:46:10.502934+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:30.892351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":342,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598692+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:46:16.598692+0000","last_peered":"2024-08-17T17:46:16.598692+0000","last_clean":"2024-08-17T17:46:16.598692+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:46:16.598692+0000","last_undegraded":"2024-08-17T17:46:16.598692+0000","last_fullsized":"2024-08-17T17:46:16.598692+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"434'7821","reported_seq":7880,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.427830+0000","last_change":"2024-08-17T17:44:36.968089+0000","last_active":"2024-08-17T17:46:26.427830+0000","last_peered":"2024-08-17T17:46:26.427830+0000","last_clean":"2024-08-17T17:46:26.427830+0000","last_became_active":"2024-08-17T17:44:36.967095+0000","last_became_peered":"2024-08-17T17:44:36.967095+0000","last_unstale":"2024-08-17T17:46:26.427830+0000","last_undegraded":"2024-08-17T17:46:26.427830+0000","last_fullsized":"2024-08-17T17:46:26.427830+0000","mapping_epoch":410,"log_start":"418'500","ondisk_log_start":"418'500","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7321,"log_dups_size":0,"ondisk_log_size":7321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:57.468862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":498401280,"num_objects":7605,"num_object_clones":0,"num_object_copies":22815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7821,"num_write_kb":493632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":533,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603562+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:46:16.603562+0000","last_peered":"2024-08-17T17:46:16.603562+0000","last_clean":"2024-08-17T17:46:16.603562+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:46:16.603562+0000","last_undegraded":"2024-08-17T17:46:16.603562+0000","last_fullsized":"2024-08-17T17:46:16.603562+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":815,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502416+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:46:10.502416+0000","last_peered":"2024-08-17T17:46:10.502416+0000","last_clean":"2024-08-17T17:46:10.502416+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:46:10.502416+0000","last_undegraded":"2024-08-17T17:46:10.502416+0000","last_fullsized":"2024-08-17T17:46:10.502416+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":744,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594589+0000","last_change":"2024-08-17T17:44:45.659724+0000","last_active":"2024-08-17T17:46:16.594589+0000","last_peered":"2024-08-17T17:46:16.594589+0000","last_clean":"2024-08-17T17:46:16.594589+0000","last_became_active":"2024-08-17T17:44:45.654153+0000","last_became_peered":"2024-08-17T17:44:45.654153+0000","last_unstale":"2024-08-17T17:46:16.594589+0000","last_undegraded":"2024-08-17T17:46:16.594589+0000","last_fullsized":"2024-08-17T17:46:16.594589+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.165904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":734,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599826+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:46:16.599826+0000","last_peered":"2024-08-17T17:46:16.599826+0000","last_clean":"2024-08-17T17:46:16.599826+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:46:16.599826+0000","last_undegraded":"2024-08-17T17:46:16.599826+0000","last_fullsized":"2024-08-17T17:46:16.599826+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"434'7885","reported_seq":7918,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.606056+0000","last_change":"2024-08-17T17:44:36.966264+0000","last_active":"2024-08-17T17:46:25.606056+0000","last_peered":"2024-08-17T17:46:25.606056+0000","last_clean":"2024-08-17T17:46:25.606056+0000","last_became_active":"2024-08-17T17:44:36.965940+0000","last_became_peered":"2024-08-17T17:44:36.965940+0000","last_unstale":"2024-08-17T17:46:25.606056+0000","last_undegraded":"2024-08-17T17:46:25.606056+0000","last_fullsized":"2024-08-17T17:46:25.606056+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7885,"log_dups_size":0,"ondisk_log_size":7885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:16.157942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":509018112,"num_objects":7767,"num_object_clones":0,"num_object_copies":23301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7885,"num_write_kb":500864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":500,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.503351+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:46:10.503351+0000","last_peered":"2024-08-17T17:46:10.503351+0000","last_clean":"2024-08-17T17:46:10.503351+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:46:10.503351+0000","last_undegraded":"2024-08-17T17:46:10.503351+0000","last_fullsized":"2024-08-17T17:46:10.503351+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":480,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.595136+0000","last_change":"2024-08-17T17:44:45.679401+0000","last_active":"2024-08-17T17:46:16.595136+0000","last_peered":"2024-08-17T17:46:16.595136+0000","last_clean":"2024-08-17T17:46:16.595136+0000","last_became_active":"2024-08-17T17:44:45.664192+0000","last_became_peered":"2024-08-17T17:44:45.664192+0000","last_unstale":"2024-08-17T17:46:16.595136+0000","last_undegraded":"2024-08-17T17:46:16.595136+0000","last_fullsized":"2024-08-17T17:46:16.595136+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:21.066280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":727,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603312+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:46:16.603312+0000","last_peered":"2024-08-17T17:46:16.603312+0000","last_clean":"2024-08-17T17:46:16.603312+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:46:16.603312+0000","last_undegraded":"2024-08-17T17:46:16.603312+0000","last_fullsized":"2024-08-17T17:46:16.603312+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":774,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.604321+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:46:16.604321+0000","last_peered":"2024-08-17T17:46:16.604321+0000","last_clean":"2024-08-17T17:46:16.604321+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:46:16.604321+0000","last_undegraded":"2024-08-17T17:46:16.604321+0000","last_fullsized":"2024-08-17T17:46:16.604321+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"434'7919","reported_seq":7952,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:24.792790+0000","last_change":"2024-08-17T17:44:36.610536+0000","last_active":"2024-08-17T17:46:24.792790+0000","last_peered":"2024-08-17T17:46:24.792790+0000","last_clean":"2024-08-17T17:46:24.792790+0000","last_became_active":"2024-08-17T17:44:36.610195+0000","last_became_peered":"2024-08-17T17:44:36.610195+0000","last_unstale":"2024-08-17T17:46:24.792790+0000","last_undegraded":"2024-08-17T17:46:24.792790+0000","last_fullsized":"2024-08-17T17:46:24.792790+0000","mapping_epoch":410,"log_start":"420'1600","ondisk_log_start":"420'1600","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6319,"log_dups_size":0,"ondisk_log_size":6319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:16.686185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":518848512,"num_objects":7917,"num_object_clones":0,"num_object_copies":23751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7919,"num_write_kb":506752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":451,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502280+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:46:10.502280+0000","last_peered":"2024-08-17T17:46:10.502280+0000","last_clean":"2024-08-17T17:46:10.502280+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:46:10.502280+0000","last_undegraded":"2024-08-17T17:46:10.502280+0000","last_fullsized":"2024-08-17T17:46:10.502280+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1151,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.503055+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:46:10.503055+0000","last_peered":"2024-08-17T17:46:10.503055+0000","last_clean":"2024-08-17T17:46:10.503055+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:46:10.503055+0000","last_undegraded":"2024-08-17T17:46:10.503055+0000","last_fullsized":"2024-08-17T17:46:10.503055+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":552,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.593311+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:46:16.593311+0000","last_peered":"2024-08-17T17:46:16.593311+0000","last_clean":"2024-08-17T17:46:16.593311+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:46:16.593311+0000","last_undegraded":"2024-08-17T17:46:16.593311+0000","last_fullsized":"2024-08-17T17:46:16.593311+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500352+0000","last_change":"2024-08-17T17:44:45.659123+0000","last_active":"2024-08-17T17:46:10.500352+0000","last_peered":"2024-08-17T17:46:10.500352+0000","last_clean":"2024-08-17T17:46:10.500352+0000","last_became_active":"2024-08-17T17:44:45.658785+0000","last_became_peered":"2024-08-17T17:44:45.658785+0000","last_unstale":"2024-08-17T17:46:10.500352+0000","last_undegraded":"2024-08-17T17:46:10.500352+0000","last_fullsized":"2024-08-17T17:46:10.500352+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:30.466001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"434'7942","reported_seq":7975,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.002920+0000","last_change":"2024-08-17T17:44:36.607785+0000","last_active":"2024-08-17T17:46:26.002920+0000","last_peered":"2024-08-17T17:46:26.002920+0000","last_clean":"2024-08-17T17:46:26.002920+0000","last_became_active":"2024-08-17T17:44:36.607472+0000","last_became_peered":"2024-08-17T17:44:36.607472+0000","last_unstale":"2024-08-17T17:46:26.002920+0000","last_undegraded":"2024-08-17T17:46:26.002920+0000","last_fullsized":"2024-08-17T17:46:26.002920+0000","mapping_epoch":410,"log_start":"418'400","ondisk_log_start":"418'400","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7542,"log_dups_size":0,"ondisk_log_size":7542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:39.734707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":508821504,"num_objects":7764,"num_object_clones":0,"num_object_copies":23292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7942,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":745,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504383+0000","last_change":"2024-08-17T17:44:52.091382+0000","last_active":"2024-08-17T17:46:10.504383+0000","last_peered":"2024-08-17T17:46:10.504383+0000","last_clean":"2024-08-17T17:46:10.504383+0000","last_became_active":"2024-08-17T17:44:48.663244+0000","last_became_peered":"2024-08-17T17:44:48.663244+0000","last_unstale":"2024-08-17T17:46:10.504383+0000","last_undegraded":"2024-08-17T17:46:10.504383+0000","last_fullsized":"2024-08-17T17:46:10.504383+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:44:52.091249+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:18.054213+0000","scrub_duration":0.001820239,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":229,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500662+0000","last_change":"2024-08-17T17:44:45.653860+0000","last_active":"2024-08-17T17:46:10.500662+0000","last_peered":"2024-08-17T17:46:10.500662+0000","last_clean":"2024-08-17T17:46:10.500662+0000","last_became_active":"2024-08-17T17:44:45.652151+0000","last_became_peered":"2024-08-17T17:44:45.652151+0000","last_unstale":"2024-08-17T17:46:10.500662+0000","last_undegraded":"2024-08-17T17:46:10.500662+0000","last_fullsized":"2024-08-17T17:46:10.500662+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:17.052902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":1019,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603663+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:46:16.603663+0000","last_peered":"2024-08-17T17:46:16.603663+0000","last_clean":"2024-08-17T17:46:16.603663+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:46:16.603663+0000","last_undegraded":"2024-08-17T17:46:16.603663+0000","last_fullsized":"2024-08-17T17:46:16.603663+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":743,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.605214+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:46:16.605214+0000","last_peered":"2024-08-17T17:46:16.605214+0000","last_clean":"2024-08-17T17:46:16.605214+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:46:16.605214+0000","last_undegraded":"2024-08-17T17:46:16.605214+0000","last_fullsized":"2024-08-17T17:46:16.605214+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"434'7837","reported_seq":7871,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.040377+0000","last_change":"2024-08-17T17:44:36.611239+0000","last_active":"2024-08-17T17:46:26.040377+0000","last_peered":"2024-08-17T17:46:26.040377+0000","last_clean":"2024-08-17T17:46:26.040377+0000","last_became_active":"2024-08-17T17:44:36.610907+0000","last_became_peered":"2024-08-17T17:44:36.610907+0000","last_unstale":"2024-08-17T17:46:26.040377+0000","last_undegraded":"2024-08-17T17:46:26.040377+0000","last_fullsized":"2024-08-17T17:46:26.040377+0000","mapping_epoch":410,"log_start":"420'1700","ondisk_log_start":"420'1700","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6137,"log_dups_size":0,"ondisk_log_size":6137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:43.474878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504234008,"num_objects":7695,"num_object_clones":0,"num_object_copies":23085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7695,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7837,"num_write_kb":496961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":546,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599662+0000","last_change":"2024-08-17T17:44:48.665077+0000","last_active":"2024-08-17T17:46:16.599662+0000","last_peered":"2024-08-17T17:46:16.599662+0000","last_clean":"2024-08-17T17:46:16.599662+0000","last_became_active":"2024-08-17T17:44:48.664818+0000","last_became_peered":"2024-08-17T17:44:48.664818+0000","last_unstale":"2024-08-17T17:46:16.599662+0000","last_undegraded":"2024-08-17T17:46:16.599662+0000","last_fullsized":"2024-08-17T17:46:16.599662+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:36.058763+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":740,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603572+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:46:16.603572+0000","last_peered":"2024-08-17T17:46:16.603572+0000","last_clean":"2024-08-17T17:46:16.603572+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:46:16.603572+0000","last_undegraded":"2024-08-17T17:46:16.603572+0000","last_fullsized":"2024-08-17T17:46:16.603572+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1005,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534290+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:46:16.534290+0000","last_peered":"2024-08-17T17:46:16.534290+0000","last_clean":"2024-08-17T17:46:16.534290+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:46:16.534290+0000","last_undegraded":"2024-08-17T17:46:16.534290+0000","last_fullsized":"2024-08-17T17:46:16.534290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":737,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.595121+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:46:16.595121+0000","last_peered":"2024-08-17T17:46:16.595121+0000","last_clean":"2024-08-17T17:46:16.595121+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:46:16.595121+0000","last_undegraded":"2024-08-17T17:46:16.595121+0000","last_fullsized":"2024-08-17T17:46:16.595121+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"434'7935","reported_seq":7968,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.996057+0000","last_change":"2024-08-17T17:44:36.966926+0000","last_active":"2024-08-17T17:46:25.996057+0000","last_peered":"2024-08-17T17:46:25.996057+0000","last_clean":"2024-08-17T17:46:25.996057+0000","last_became_active":"2024-08-17T17:44:36.966608+0000","last_became_peered":"2024-08-17T17:44:36.966608+0000","last_unstale":"2024-08-17T17:46:25.996057+0000","last_undegraded":"2024-08-17T17:46:25.996057+0000","last_fullsized":"2024-08-17T17:46:25.996057+0000","mapping_epoch":410,"log_start":"420'1800","ondisk_log_start":"420'1800","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:39.384622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":510328832,"num_objects":7787,"num_object_clones":0,"num_object_copies":23361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7935,"num_write_kb":503104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":452,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507522+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:46:10.507522+0000","last_peered":"2024-08-17T17:46:10.507522+0000","last_clean":"2024-08-17T17:46:10.507522+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:46:10.507522+0000","last_undegraded":"2024-08-17T17:46:10.507522+0000","last_fullsized":"2024-08-17T17:46:10.507522+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":56,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504556+0000","last_change":"2024-08-17T17:44:45.670035+0000","last_active":"2024-08-17T17:46:10.504556+0000","last_peered":"2024-08-17T17:46:10.504556+0000","last_clean":"2024-08-17T17:46:10.504556+0000","last_became_active":"2024-08-17T17:44:45.655566+0000","last_became_peered":"2024-08-17T17:44:45.655566+0000","last_unstale":"2024-08-17T17:46:10.504556+0000","last_undegraded":"2024-08-17T17:46:10.504556+0000","last_fullsized":"2024-08-17T17:46:10.504556+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:17:32.230235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1250,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.604103+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:46:16.604103+0000","last_peered":"2024-08-17T17:46:16.604103+0000","last_clean":"2024-08-17T17:46:16.604103+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:46:16.604103+0000","last_undegraded":"2024-08-17T17:46:16.604103+0000","last_fullsized":"2024-08-17T17:46:16.604103+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.502170+0000","last_change":"2024-08-17T17:44:45.656210+0000","last_active":"2024-08-17T17:46:10.502170+0000","last_peered":"2024-08-17T17:46:10.502170+0000","last_clean":"2024-08-17T17:46:10.502170+0000","last_became_active":"2024-08-17T17:44:45.655853+0000","last_became_peered":"2024-08-17T17:44:45.655853+0000","last_unstale":"2024-08-17T17:46:10.502170+0000","last_undegraded":"2024-08-17T17:46:10.502170+0000","last_fullsized":"2024-08-17T17:46:10.502170+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:01:57.057139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"434'7817","reported_seq":7859,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:24.767092+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:24.767092+0000","last_peered":"2024-08-17T17:46:24.767092+0000","last_clean":"2024-08-17T17:46:24.767092+0000","last_became_active":"2024-08-17T17:44:45.658966+0000","last_became_peered":"2024-08-17T17:44:45.658966+0000","last_unstale":"2024-08-17T17:46:24.767092+0000","last_undegraded":"2024-08-17T17:46:24.767092+0000","last_fullsized":"2024-08-17T17:46:24.767092+0000","mapping_epoch":413,"log_start":"420'1500","ondisk_log_start":"420'1500","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6317,"log_dups_size":0,"ondisk_log_size":6317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:22.822524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":512163840,"num_objects":7815,"num_object_clones":0,"num_object_copies":23445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7817,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":420,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.500431+0000","last_change":"2024-08-17T17:44:51.112004+0000","last_active":"2024-08-17T17:46:10.500431+0000","last_peered":"2024-08-17T17:46:10.500431+0000","last_clean":"2024-08-17T17:46:10.500431+0000","last_became_active":"2024-08-17T17:44:48.661437+0000","last_became_peered":"2024-08-17T17:44:48.661437+0000","last_unstale":"2024-08-17T17:46:10.500431+0000","last_undegraded":"2024-08-17T17:46:10.500431+0000","last_fullsized":"2024-08-17T17:46:10.500431+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:00.448235+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":749,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599717+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:46:16.599717+0000","last_peered":"2024-08-17T17:46:16.599717+0000","last_clean":"2024-08-17T17:46:16.599717+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:46:16.599717+0000","last_undegraded":"2024-08-17T17:46:16.599717+0000","last_fullsized":"2024-08-17T17:46:16.599717+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":68,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504301+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:46:10.504301+0000","last_peered":"2024-08-17T17:46:10.504301+0000","last_clean":"2024-08-17T17:46:10.504301+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:46:10.504301+0000","last_undegraded":"2024-08-17T17:46:10.504301+0000","last_fullsized":"2024-08-17T17:46:10.504301+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":379,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.508201+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:46:10.508201+0000","last_peered":"2024-08-17T17:46:10.508201+0000","last_clean":"2024-08-17T17:46:10.508201+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:46:10.508201+0000","last_undegraded":"2024-08-17T17:46:10.508201+0000","last_fullsized":"2024-08-17T17:46:10.508201+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":342,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507525+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:46:10.507525+0000","last_peered":"2024-08-17T17:46:10.507525+0000","last_clean":"2024-08-17T17:46:10.507525+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:46:10.507525+0000","last_undegraded":"2024-08-17T17:46:10.507525+0000","last_fullsized":"2024-08-17T17:46:10.507525+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":352,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.513194+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:46:10.513194+0000","last_peered":"2024-08-17T17:46:10.513194+0000","last_clean":"2024-08-17T17:46:10.513194+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:46:10.513194+0000","last_undegraded":"2024-08-17T17:46:10.513194+0000","last_fullsized":"2024-08-17T17:46:10.513194+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":725,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.605083+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:46:16.605083+0000","last_peered":"2024-08-17T17:46:16.605083+0000","last_clean":"2024-08-17T17:46:16.605083+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:46:16.605083+0000","last_undegraded":"2024-08-17T17:46:16.605083+0000","last_fullsized":"2024-08-17T17:46:16.605083+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":451,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.505879+0000","last_change":"2024-08-17T17:44:45.668086+0000","last_active":"2024-08-17T17:46:10.505879+0000","last_peered":"2024-08-17T17:46:10.505879+0000","last_clean":"2024-08-17T17:46:10.505879+0000","last_became_active":"2024-08-17T17:44:45.654205+0000","last_became_peered":"2024-08-17T17:44:45.654205+0000","last_unstale":"2024-08-17T17:46:10.505879+0000","last_undegraded":"2024-08-17T17:46:10.505879+0000","last_fullsized":"2024-08-17T17:46:10.505879+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:30.323982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.505239+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:46:10.505239+0000","last_peered":"2024-08-17T17:46:10.505239+0000","last_clean":"2024-08-17T17:46:10.505239+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:46:10.505239+0000","last_undegraded":"2024-08-17T17:46:10.505239+0000","last_fullsized":"2024-08-17T17:46:10.505239+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":336,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.504853+0000","last_change":"2024-08-17T17:44:45.652198+0000","last_active":"2024-08-17T17:46:10.504853+0000","last_peered":"2024-08-17T17:46:10.504853+0000","last_clean":"2024-08-17T17:46:10.504853+0000","last_became_active":"2024-08-17T17:44:45.651854+0000","last_became_peered":"2024-08-17T17:44:45.651854+0000","last_unstale":"2024-08-17T17:46:10.504853+0000","last_undegraded":"2024-08-17T17:46:10.504853+0000","last_fullsized":"2024-08-17T17:46:10.504853+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:49:58.681636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":758,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.595141+0000","last_change":"2024-08-17T17:44:45.651808+0000","last_active":"2024-08-17T17:46:16.595141+0000","last_peered":"2024-08-17T17:46:16.595141+0000","last_clean":"2024-08-17T17:46:16.595141+0000","last_became_active":"2024-08-17T17:44:45.651478+0000","last_became_peered":"2024-08-17T17:44:45.651478+0000","last_unstale":"2024-08-17T17:46:16.595141+0000","last_undegraded":"2024-08-17T17:46:16.595141+0000","last_fullsized":"2024-08-17T17:46:16.595141+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:04.589247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":406,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603030+0000","last_change":"2024-08-17T17:44:45.661728+0000","last_active":"2024-08-17T17:46:16.603030+0000","last_peered":"2024-08-17T17:46:16.603030+0000","last_clean":"2024-08-17T17:46:16.603030+0000","last_became_active":"2024-08-17T17:44:45.661337+0000","last_became_peered":"2024-08-17T17:44:45.661337+0000","last_unstale":"2024-08-17T17:46:16.603030+0000","last_undegraded":"2024-08-17T17:46:16.603030+0000","last_fullsized":"2024-08-17T17:46:16.603030+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:54:21.723874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":230,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.503496+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:46:10.503496+0000","last_peered":"2024-08-17T17:46:10.503496+0000","last_clean":"2024-08-17T17:46:10.503496+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:46:10.503496+0000","last_undegraded":"2024-08-17T17:46:10.503496+0000","last_fullsized":"2024-08-17T17:46:10.503496+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594617+0000","last_change":"2024-08-17T17:44:45.663565+0000","last_active":"2024-08-17T17:46:16.594617+0000","last_peered":"2024-08-17T17:46:16.594617+0000","last_clean":"2024-08-17T17:46:16.594617+0000","last_became_active":"2024-08-17T17:44:45.663237+0000","last_became_peered":"2024-08-17T17:44:45.663237+0000","last_unstale":"2024-08-17T17:46:16.594617+0000","last_undegraded":"2024-08-17T17:46:16.594617+0000","last_fullsized":"2024-08-17T17:46:16.594617+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:26:34.711189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":731,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599821+0000","last_change":"2024-08-17T17:44:45.662891+0000","last_active":"2024-08-17T17:46:16.599821+0000","last_peered":"2024-08-17T17:46:16.599821+0000","last_clean":"2024-08-17T17:46:16.599821+0000","last_became_active":"2024-08-17T17:44:45.662580+0000","last_became_peered":"2024-08-17T17:44:45.662580+0000","last_unstale":"2024-08-17T17:46:16.599821+0000","last_undegraded":"2024-08-17T17:46:16.599821+0000","last_fullsized":"2024-08-17T17:46:16.599821+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:48.627337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":734,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603155+0000","last_change":"2024-08-17T17:44:45.657935+0000","last_active":"2024-08-17T17:46:16.603155+0000","last_peered":"2024-08-17T17:46:16.603155+0000","last_clean":"2024-08-17T17:46:16.603155+0000","last_became_active":"2024-08-17T17:44:45.657525+0000","last_became_peered":"2024-08-17T17:44:45.657525+0000","last_unstale":"2024-08-17T17:46:16.603155+0000","last_undegraded":"2024-08-17T17:46:16.603155+0000","last_fullsized":"2024-08-17T17:46:16.603155+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:55.691041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603510+0000","last_change":"2024-08-17T17:44:45.661307+0000","last_active":"2024-08-17T17:46:16.603510+0000","last_peered":"2024-08-17T17:46:16.603510+0000","last_clean":"2024-08-17T17:46:16.603510+0000","last_became_active":"2024-08-17T17:44:45.660976+0000","last_became_peered":"2024-08-17T17:44:45.660976+0000","last_unstale":"2024-08-17T17:46:16.603510+0000","last_undegraded":"2024-08-17T17:46:16.603510+0000","last_fullsized":"2024-08-17T17:46:16.603510+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:22.725842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":326,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.509331+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:46:10.509331+0000","last_peered":"2024-08-17T17:46:10.509331+0000","last_clean":"2024-08-17T17:46:10.509331+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:46:10.509331+0000","last_undegraded":"2024-08-17T17:46:10.509331+0000","last_fullsized":"2024-08-17T17:46:10.509331+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":262,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603853+0000","last_change":"2024-08-17T17:44:45.660322+0000","last_active":"2024-08-17T17:46:16.603853+0000","last_peered":"2024-08-17T17:46:16.603853+0000","last_clean":"2024-08-17T17:46:16.603853+0000","last_became_active":"2024-08-17T17:44:45.660018+0000","last_became_peered":"2024-08-17T17:44:45.660018+0000","last_unstale":"2024-08-17T17:46:16.603853+0000","last_undegraded":"2024-08-17T17:46:16.603853+0000","last_fullsized":"2024-08-17T17:46:16.603853+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:57.732705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":118,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594828+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:46:16.594828+0000","last_peered":"2024-08-17T17:46:16.594828+0000","last_clean":"2024-08-17T17:46:16.594828+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:46:16.594828+0000","last_undegraded":"2024-08-17T17:46:16.594828+0000","last_fullsized":"2024-08-17T17:46:16.594828+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":335,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507523+0000","last_change":"2024-08-17T17:44:45.657401+0000","last_active":"2024-08-17T17:46:10.507523+0000","last_peered":"2024-08-17T17:46:10.507523+0000","last_clean":"2024-08-17T17:46:10.507523+0000","last_became_active":"2024-08-17T17:44:45.657074+0000","last_became_peered":"2024-08-17T17:44:45.657074+0000","last_unstale":"2024-08-17T17:46:10.507523+0000","last_undegraded":"2024-08-17T17:46:10.507523+0000","last_fullsized":"2024-08-17T17:46:10.507523+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:19.924661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":223,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.586702+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:46:15.586702+0000","last_peered":"2024-08-17T17:46:15.586702+0000","last_clean":"2024-08-17T17:46:15.586702+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:46:15.586702+0000","last_undegraded":"2024-08-17T17:46:15.586702+0000","last_fullsized":"2024-08-17T17:46:15.586702+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":335,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.509240+0000","last_change":"2024-08-17T17:44:45.657702+0000","last_active":"2024-08-17T17:46:10.509240+0000","last_peered":"2024-08-17T17:46:10.509240+0000","last_clean":"2024-08-17T17:46:10.509240+0000","last_became_active":"2024-08-17T17:44:45.657367+0000","last_became_peered":"2024-08-17T17:44:45.657367+0000","last_unstale":"2024-08-17T17:46:10.509240+0000","last_undegraded":"2024-08-17T17:46:10.509240+0000","last_fullsized":"2024-08-17T17:46:10.509240+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:29.447263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":233,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.586738+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:46:15.586738+0000","last_peered":"2024-08-17T17:46:15.586738+0000","last_clean":"2024-08-17T17:46:15.586738+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:46:15.586738+0000","last_undegraded":"2024-08-17T17:46:15.586738+0000","last_fullsized":"2024-08-17T17:46:15.586738+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594684+0000","last_change":"2024-08-17T17:44:45.660616+0000","last_active":"2024-08-17T17:46:16.594684+0000","last_peered":"2024-08-17T17:46:16.594684+0000","last_clean":"2024-08-17T17:46:16.594684+0000","last_became_active":"2024-08-17T17:44:45.660198+0000","last_became_peered":"2024-08-17T17:44:45.660198+0000","last_unstale":"2024-08-17T17:46:16.594684+0000","last_undegraded":"2024-08-17T17:46:16.594684+0000","last_fullsized":"2024-08-17T17:46:16.594684+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:06.005702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":52,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.603711+0000","last_change":"2024-08-17T17:44:45.662070+0000","last_active":"2024-08-17T17:46:16.603711+0000","last_peered":"2024-08-17T17:46:16.603711+0000","last_clean":"2024-08-17T17:46:16.603711+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:16.603711+0000","last_undegraded":"2024-08-17T17:46:16.603711+0000","last_fullsized":"2024-08-17T17:46:16.603711+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:11.915663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":341,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-08-17T17:46:10.507248+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:46:10.507248+0000","last_peered":"2024-08-17T17:46:10.507248+0000","last_clean":"2024-08-17T17:46:10.507248+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:46:10.507248+0000","last_undegraded":"2024-08-17T17:46:10.507248+0000","last_fullsized":"2024-08-17T17:46:10.507248+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":768,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.594987+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:46:16.594987+0000","last_peered":"2024-08-17T17:46:16.594987+0000","last_clean":"2024-08-17T17:46:16.594987+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:46:16.594987+0000","last_undegraded":"2024-08-17T17:46:16.594987+0000","last_fullsized":"2024-08-17T17:46:16.594987+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":8117551128,"num_objects":123865,"num_object_clones":0,"num_object_copies":371595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":123865,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126093,"num_write_kb":7998593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24313802752,"data_stored":24313790536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111993,"ondisk_log_size":111993,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201795,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3410112,"kb_used_data":2494648,"kb_used_omap":1,"kb_used_meta":915454,"kb_avail":90335040,"statfs":{"total":95995035648,"available":92503080960,"internally_reserved":0,"allocated":2554519552,"data_stored":2552405519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":937425355},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922291,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3985460,"kb_used_data":3504876,"kb_used_omap":7,"kb_used_meta":480568,"kb_avail":89759692,"statfs":{"total":95995035648,"available":91913924608,"internally_reserved":0,"allocated":3588993024,"data_stored":3586890789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":492102326},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085815,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3107828,"kb_used_data":2507180,"kb_used_omap":7,"kb_used_meta":600568,"kb_avail":90637324,"statfs":{"total":95995035648,"available":92812619776,"internally_reserved":0,"allocated":2567352320,"data_stored":2565246818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":614982073},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986705,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5869888,"kb_used_data":4990520,"kb_used_omap":1,"kb_used_meta":879358,"kb_avail":87875264,"statfs":{"total":95995035648,"available":89984270336,"internally_reserved":0,"allocated":5110292480,"data_stored":5108178409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":900463046},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":468151435596,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599123,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4413484,"kb_used_data":3461348,"kb_used_omap":7,"kb_used_meta":952120,"kb_avail":89331668,"statfs":{"total":95995035648,"available":91475628032,"internally_reserved":0,"allocated":3544420352,"data_stored":3542321830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":974971053},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762649,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3100128,"kb_used_data":2252120,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":90645024,"statfs":{"total":95995035648,"available":92820504576,"internally_reserved":0,"allocated":2306170880,"data_stored":2304083232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":868344225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926175,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5955576,"kb_used_data":4981168,"kb_used_omap":7,"kb_used_meta":974328,"kb_avail":87789576,"statfs":{"total":95995035648,"available":89896525824,"internally_reserved":0,"allocated":5100716032,"data_stored":5098606230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":997712476},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":5047717888,"data_stored":5047713816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2019622912,"data_stored":2019622912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3517972480,"data_stored":3517972480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":5069869056,"data_stored":5069864984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2545418240,"data_stored":2545418240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3587379200,"data_stored":3587375128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2525822976,"data_stored":2525822976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:46:29.468 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:46:29.468 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:46:29.468 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T17:46:29.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:46:29.477 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:29.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:29.612 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:46:29.617 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:29.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[104516]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:29.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3551995804' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[109246]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:29.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3551995804' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.712 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:29.712 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:29.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:29.753 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:29.855 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:29.869 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:29 smithi175 ceph-mon[101434]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:29.869 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3551995804' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.869 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:29.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:29.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.029 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:30.130 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:30.165 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:30.267 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:30.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:30.432 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:30.533 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:30.554 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:30.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:30.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:46:30.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:30.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.795 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:30.816 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:30 smithi175 ceph-mon[101434]: pgmap v1399: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 85 B/s rd, 45 MiB/s wr, 808 op/s 2024-08-17T17:46:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:30 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:30 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:46:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:30 smithi175 ceph-mon[101434]: osdmap e435: 8 total, 8 up, 7 in 2024-08-17T17:46:30.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[104516]: pgmap v1399: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 85 B/s rd, 45 MiB/s wr, 808 op/s 2024-08-17T17:46:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:46:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:30.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[104516]: osdmap e435: 8 total, 8 up, 7 in 2024-08-17T17:46:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[109246]: pgmap v1399: 99 pgs: 99 active+clean; 7.6 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 85 B/s rd, 45 MiB/s wr, 808 op/s 2024-08-17T17:46:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:46:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:30.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:30 smithi064 ceph-mon[109246]: osdmap e435: 8 total, 8 up, 7 in 2024-08-17T17:46:30.940 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:30.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:31.042 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:31.064 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:31.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:31.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:31.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:31.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.333 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:31.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:31.434 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:31.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:31.571 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:31.597 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:31.698 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:31.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:31.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:31.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:31.965 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:32.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.002 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:32.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:32.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:32.143 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:32.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:32.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:32.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:32.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:32.427 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:32.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:32.566 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:32.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:32.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:32.707 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:32.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:32.845 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:32.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:32 smithi175 ceph-mon[101434]: pgmap v1401: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 34 MiB/s wr, 994 op/s 2024-08-17T17:46:32.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:32 smithi064 ceph-mon[104516]: pgmap v1401: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 34 MiB/s wr, 994 op/s 2024-08-17T17:46:32.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:32 smithi064 ceph-mon[109246]: pgmap v1401: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 34 MiB/s wr, 994 op/s 2024-08-17T17:46:32.938 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:32.946 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:32.950 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:32.978 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:33.079 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:33.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:33.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:46:33.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:33.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:33.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.388 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:33.488 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:33.524 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:33.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:33.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:33.768 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:33.813 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:33.915 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:33.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:34.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:34.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:34.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:34.219 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:34.319 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:33] "GET /metrics HTTP/1.1" 200 36700 "" "Prometheus/2.51.0" 2024-08-17T17:46:34.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:34.363 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:34.465 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:34.500 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:34.602 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:34.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:34.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:34.779 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:34.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:34.881 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:34.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:34 smithi064 ceph-mon[104516]: pgmap v1402: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 978 op/s 2024-08-17T17:46:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:34 smithi064 ceph-mon[109246]: pgmap v1402: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 978 op/s 2024-08-17T17:46:34.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:34 smithi175 ceph-mon[101434]: pgmap v1402: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 25 MiB/s wr, 978 op/s 2024-08-17T17:46:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.917 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:34.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:35.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:35.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:35.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.198 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:35.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:35.280 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:46:35.282 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:46:35.299 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:35.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:35.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.478 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:35.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:35.579 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:35.622 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:35.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:35.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:35.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:35.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.896 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:35.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:35.996 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:36.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:36.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:36.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:36.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:36.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:36.399 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:36.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:36.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:36.579 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:36.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:36.715 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:36.817 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:36.849 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:36.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:36 smithi064 ceph-mon[109246]: from='client.46253 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:36.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:36 smithi064 ceph-mon[109246]: pgmap v1403: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 951 op/s 2024-08-17T17:46:36.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:36 smithi064 ceph-mon[104516]: from='client.46253 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:36.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:36 smithi064 ceph-mon[104516]: pgmap v1403: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 951 op/s 2024-08-17T17:46:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:36 smithi175 ceph-mon[101434]: from='client.46253 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:36 smithi175 ceph-mon[101434]: pgmap v1403: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 951 op/s 2024-08-17T17:46:36.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:36.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:37.047 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:37.047 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:37.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:37.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:37.221 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:37.251 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:37.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:37.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:37.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:37.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:37.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:37.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:37.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:37.797 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:37.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:37.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:38.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:38.069 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:38.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:38.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:38.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:38.335 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:38.437 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:38.468 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1402,"stamp":"2024-08-17T17:46:33.374201+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7813208061,"num_objects":119176,"num_object_clones":0,"num_object_copies":357528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":119176,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":131718,"num_write_kb":8011218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8657757,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115982,"ondisk_log_size":115982,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":28870080,"kb_used_data":23206912,"kb_used_omap":40,"kb_used_meta":5662807,"kb_avail":627345984,"statfs":{"total":671965249536,"available":642402287616,"internally_reserved":0,"allocated":23763877888,"data_stored":23749118357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41814,"internal_metadata":5798714538},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123142120,"num_objects":-1878,"num_object_clones":0,"num_object_copies":-5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1878,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9784,"num_write_kb":252929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002763"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":117,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629332+0000","last_change":"2024-08-17T17:44:45.659658+0000","last_active":"2024-08-17T17:46:29.629332+0000","last_peered":"2024-08-17T17:46:29.629332+0000","last_clean":"2024-08-17T17:46:29.629332+0000","last_became_active":"2024-08-17T17:44:45.659189+0000","last_became_peered":"2024-08-17T17:44:45.659189+0000","last_unstale":"2024-08-17T17:46:29.629332+0000","last_undegraded":"2024-08-17T17:46:29.629332+0000","last_fullsized":"2024-08-17T17:46:29.629332+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:45.422590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":437,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533227+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:46:16.533227+0000","last_peered":"2024-08-17T17:46:16.533227+0000","last_clean":"2024-08-17T17:46:16.533227+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:46:16.533227+0000","last_undegraded":"2024-08-17T17:46:16.533227+0000","last_fullsized":"2024-08-17T17:46:16.533227+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533814+0000","last_change":"2024-08-17T17:44:45.650591+0000","last_active":"2024-08-17T17:46:16.533814+0000","last_peered":"2024-08-17T17:46:16.533814+0000","last_clean":"2024-08-17T17:46:16.533814+0000","last_became_active":"2024-08-17T17:44:45.650249+0000","last_became_peered":"2024-08-17T17:44:45.650249+0000","last_unstale":"2024-08-17T17:46:16.533814+0000","last_undegraded":"2024-08-17T17:46:16.533814+0000","last_fullsized":"2024-08-17T17:46:16.533814+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:06.858698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":731,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.631441+0000","last_change":"2024-08-17T17:44:45.663771+0000","last_active":"2024-08-17T17:46:29.631441+0000","last_peered":"2024-08-17T17:46:29.631441+0000","last_clean":"2024-08-17T17:46:29.631441+0000","last_became_active":"2024-08-17T17:44:45.663321+0000","last_became_peered":"2024-08-17T17:44:45.663321+0000","last_unstale":"2024-08-17T17:46:29.631441+0000","last_undegraded":"2024-08-17T17:46:29.631441+0000","last_fullsized":"2024-08-17T17:46:29.631441+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:30.374220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":731,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630457+0000","last_change":"2024-08-17T17:44:45.657665+0000","last_active":"2024-08-17T17:46:29.630457+0000","last_peered":"2024-08-17T17:46:29.630457+0000","last_clean":"2024-08-17T17:46:29.630457+0000","last_became_active":"2024-08-17T17:44:45.657342+0000","last_became_peered":"2024-08-17T17:44:45.657342+0000","last_unstale":"2024-08-17T17:46:29.630457+0000","last_undegraded":"2024-08-17T17:46:29.630457+0000","last_fullsized":"2024-08-17T17:46:29.630457+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:00:18.245759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533118+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:46:16.533118+0000","last_peered":"2024-08-17T17:46:16.533118+0000","last_clean":"2024-08-17T17:46:16.533118+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:46:16.533118+0000","last_undegraded":"2024-08-17T17:46:16.533118+0000","last_fullsized":"2024-08-17T17:46:16.533118+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":95,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534255+0000","last_change":"2024-08-17T17:44:45.660525+0000","last_active":"2024-08-17T17:46:16.534255+0000","last_peered":"2024-08-17T17:46:16.534255+0000","last_clean":"2024-08-17T17:46:16.534255+0000","last_became_active":"2024-08-17T17:44:45.660152+0000","last_became_peered":"2024-08-17T17:44:45.660152+0000","last_unstale":"2024-08-17T17:46:16.534255+0000","last_undegraded":"2024-08-17T17:46:16.534255+0000","last_fullsized":"2024-08-17T17:46:16.534255+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:35:09.946483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":371,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.596749+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:46:16.596749+0000","last_peered":"2024-08-17T17:46:16.596749+0000","last_clean":"2024-08-17T17:46:16.596749+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:46:16.596749+0000","last_undegraded":"2024-08-17T17:46:16.596749+0000","last_fullsized":"2024-08-17T17:46:16.596749+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"435'8216","reported_seq":8250,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.474541+0000","last_change":"2024-08-17T17:44:36.608512+0000","last_active":"2024-08-17T17:46:30.474541+0000","last_peered":"2024-08-17T17:46:30.474541+0000","last_clean":"2024-08-17T17:46:30.474541+0000","last_became_active":"2024-08-17T17:44:36.608224+0000","last_became_peered":"2024-08-17T17:44:36.608224+0000","last_unstale":"2024-08-17T17:46:30.474541+0000","last_undegraded":"2024-08-17T17:46:30.474541+0000","last_fullsized":"2024-08-17T17:46:30.474541+0000","mapping_epoch":410,"log_start":"418'100","ondisk_log_start":"418'100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:46.388975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":492437504,"num_objects":7514,"num_object_clones":0,"num_object_copies":22542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8216,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":433,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.037756+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:46:21.037756+0000","last_peered":"2024-08-17T17:46:21.037756+0000","last_clean":"2024-08-17T17:46:21.037756+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:46:21.037756+0000","last_undegraded":"2024-08-17T17:46:21.037756+0000","last_fullsized":"2024-08-17T17:46:21.037756+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":231,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:25.626925+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:46:25.626925+0000","last_peered":"2024-08-17T17:46:25.626925+0000","last_clean":"2024-08-17T17:46:25.626925+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:46:25.626925+0000","last_undegraded":"2024-08-17T17:46:25.626925+0000","last_fullsized":"2024-08-17T17:46:25.626925+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"435'8080","reported_seq":8113,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.469984+0000","last_change":"2024-08-17T17:44:45.652247+0000","last_active":"2024-08-17T17:46:30.469984+0000","last_peered":"2024-08-17T17:46:30.469984+0000","last_clean":"2024-08-17T17:46:30.469984+0000","last_became_active":"2024-08-17T17:44:45.651926+0000","last_became_peered":"2024-08-17T17:44:45.651926+0000","last_unstale":"2024-08-17T17:46:30.469984+0000","last_undegraded":"2024-08-17T17:46:30.469984+0000","last_fullsized":"2024-08-17T17:46:30.469984+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8080,"log_dups_size":0,"ondisk_log_size":8080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:39.517297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479199232,"num_objects":7312,"num_object_clones":0,"num_object_copies":21936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8080,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":434,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534184+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:46:16.534184+0000","last_peered":"2024-08-17T17:46:16.534184+0000","last_clean":"2024-08-17T17:46:16.534184+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:46:16.534184+0000","last_undegraded":"2024-08-17T17:46:16.534184+0000","last_fullsized":"2024-08-17T17:46:16.534184+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":338,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.596420+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:16.596420+0000","last_peered":"2024-08-17T17:46:16.596420+0000","last_clean":"2024-08-17T17:46:16.596420+0000","last_became_active":"2024-08-17T17:44:45.658981+0000","last_became_peered":"2024-08-17T17:44:45.658981+0000","last_unstale":"2024-08-17T17:46:16.596420+0000","last_undegraded":"2024-08-17T17:46:16.596420+0000","last_fullsized":"2024-08-17T17:46:16.596420+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:59.247763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"435'8174","reported_seq":8241,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:31.393159+0000","last_change":"2024-08-17T17:44:45.662022+0000","last_active":"2024-08-17T17:46:31.393159+0000","last_peered":"2024-08-17T17:46:31.393159+0000","last_clean":"2024-08-17T17:46:31.393159+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:31.393159+0000","last_undegraded":"2024-08-17T17:46:31.393159+0000","last_fullsized":"2024-08-17T17:46:31.393159+0000","mapping_epoch":413,"log_start":"418'800","ondisk_log_start":"418'800","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:02.414550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":485752832,"num_objects":7412,"num_object_clones":0,"num_object_copies":22236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8174,"num_write_kb":498752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":451,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534289+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:46:16.534289+0000","last_peered":"2024-08-17T17:46:16.534289+0000","last_clean":"2024-08-17T17:46:16.534289+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:46:16.534289+0000","last_undegraded":"2024-08-17T17:46:16.534289+0000","last_fullsized":"2024-08-17T17:46:16.534289+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":351,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598001+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:46:16.598001+0000","last_peered":"2024-08-17T17:46:16.598001+0000","last_clean":"2024-08-17T17:46:16.598001+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:46:16.598001+0000","last_undegraded":"2024-08-17T17:46:16.598001+0000","last_fullsized":"2024-08-17T17:46:16.598001+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"435'8191","reported_seq":8259,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:31.415995+0000","last_change":"2024-08-17T17:44:45.658418+0000","last_active":"2024-08-17T17:46:31.415995+0000","last_peered":"2024-08-17T17:46:31.415995+0000","last_clean":"2024-08-17T17:46:31.415995+0000","last_became_active":"2024-08-17T17:44:45.658105+0000","last_became_peered":"2024-08-17T17:44:45.658105+0000","last_unstale":"2024-08-17T17:46:31.415995+0000","last_undegraded":"2024-08-17T17:46:31.415995+0000","last_fullsized":"2024-08-17T17:46:31.415995+0000","mapping_epoch":413,"log_start":"418'800","ondisk_log_start":"418'800","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:47.860621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":489095168,"num_objects":7463,"num_object_clones":0,"num_object_copies":22389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8191,"num_write_kb":500928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":239,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534059+0000","last_change":"2024-08-17T17:44:45.657383+0000","last_active":"2024-08-17T17:46:16.534059+0000","last_peered":"2024-08-17T17:46:16.534059+0000","last_clean":"2024-08-17T17:46:16.534059+0000","last_became_active":"2024-08-17T17:44:45.656436+0000","last_became_peered":"2024-08-17T17:44:45.656436+0000","last_unstale":"2024-08-17T17:46:16.534059+0000","last_undegraded":"2024-08-17T17:46:16.534059+0000","last_fullsized":"2024-08-17T17:46:16.534059+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:23:40.172334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":745,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630693+0000","last_change":"2024-08-17T17:45:27.582532+0000","last_active":"2024-08-17T17:46:29.630693+0000","last_peered":"2024-08-17T17:46:29.630693+0000","last_clean":"2024-08-17T17:46:29.630693+0000","last_became_active":"2024-08-17T17:45:27.582184+0000","last_became_peered":"2024-08-17T17:45:27.582184+0000","last_unstale":"2024-08-17T17:46:29.630693+0000","last_undegraded":"2024-08-17T17:46:29.630693+0000","last_fullsized":"2024-08-17T17:46:29.630693+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:28.064920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"435'8226","reported_seq":8287,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:31.639483+0000","last_change":"2024-08-17T17:44:36.607826+0000","last_active":"2024-08-17T17:46:31.639483+0000","last_peered":"2024-08-17T17:46:31.639483+0000","last_clean":"2024-08-17T17:46:31.639483+0000","last_became_active":"2024-08-17T17:44:36.607543+0000","last_became_peered":"2024-08-17T17:44:36.607543+0000","last_unstale":"2024-08-17T17:46:31.639483+0000","last_undegraded":"2024-08-17T17:46:31.639483+0000","last_fullsized":"2024-08-17T17:46:31.639483+0000","mapping_epoch":410,"log_start":"420'1400","ondisk_log_start":"420'1400","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6826,"log_dups_size":0,"ondisk_log_size":6826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:22.366195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486014976,"num_objects":7416,"num_object_clones":0,"num_object_copies":22248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8226,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":809,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.532966+0000","last_change":"2024-08-17T17:44:48.661664+0000","last_active":"2024-08-17T17:46:16.532966+0000","last_peered":"2024-08-17T17:46:16.532966+0000","last_clean":"2024-08-17T17:46:16.532966+0000","last_became_active":"2024-08-17T17:44:48.661277+0000","last_became_peered":"2024-08-17T17:44:48.661277+0000","last_unstale":"2024-08-17T17:46:16.532966+0000","last_undegraded":"2024-08-17T17:46:16.532966+0000","last_fullsized":"2024-08-17T17:46:16.532966+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.305103+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":81,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630528+0000","last_change":"2024-08-17T17:44:45.670332+0000","last_active":"2024-08-17T17:46:29.630528+0000","last_peered":"2024-08-17T17:46:29.630528+0000","last_clean":"2024-08-17T17:46:29.630528+0000","last_became_active":"2024-08-17T17:44:45.660508+0000","last_became_peered":"2024-08-17T17:44:45.660508+0000","last_unstale":"2024-08-17T17:46:29.630528+0000","last_undegraded":"2024-08-17T17:46:29.630528+0000","last_fullsized":"2024-08-17T17:46:29.630528+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:17.489964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":361,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599072+0000","last_change":"2024-08-17T17:44:45.656521+0000","last_active":"2024-08-17T17:46:16.599072+0000","last_peered":"2024-08-17T17:46:16.599072+0000","last_clean":"2024-08-17T17:46:16.599072+0000","last_became_active":"2024-08-17T17:44:45.656178+0000","last_became_peered":"2024-08-17T17:44:45.656178+0000","last_unstale":"2024-08-17T17:46:16.599072+0000","last_undegraded":"2024-08-17T17:46:16.599072+0000","last_fullsized":"2024-08-17T17:46:16.599072+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:48.486263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"435'8152","reported_seq":8192,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.451315+0000","last_change":"2024-08-17T17:44:45.653325+0000","last_active":"2024-08-17T17:46:30.451315+0000","last_peered":"2024-08-17T17:46:30.451315+0000","last_clean":"2024-08-17T17:46:30.451315+0000","last_became_active":"2024-08-17T17:44:45.652943+0000","last_became_peered":"2024-08-17T17:44:45.652943+0000","last_unstale":"2024-08-17T17:46:30.451315+0000","last_undegraded":"2024-08-17T17:46:30.451315+0000","last_fullsized":"2024-08-17T17:46:30.451315+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:29:49.759034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486801408,"num_objects":7428,"num_object_clones":0,"num_object_copies":22284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8152,"num_write_kb":498560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":473,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533127+0000","last_change":"2024-08-17T17:44:48.661693+0000","last_active":"2024-08-17T17:46:16.533127+0000","last_peered":"2024-08-17T17:46:16.533127+0000","last_clean":"2024-08-17T17:46:16.533127+0000","last_became_active":"2024-08-17T17:44:48.661280+0000","last_became_peered":"2024-08-17T17:44:48.661280+0000","last_unstale":"2024-08-17T17:46:16.533127+0000","last_undegraded":"2024-08-17T17:46:16.533127+0000","last_fullsized":"2024-08-17T17:46:16.533127+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:56.378196+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":358,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598317+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:46:16.598317+0000","last_peered":"2024-08-17T17:46:16.598317+0000","last_clean":"2024-08-17T17:46:16.598317+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:46:16.598317+0000","last_undegraded":"2024-08-17T17:46:16.598317+0000","last_fullsized":"2024-08-17T17:46:16.598317+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":328,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598773+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:46:16.598773+0000","last_peered":"2024-08-17T17:46:16.598773+0000","last_clean":"2024-08-17T17:46:16.598773+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:46:16.598773+0000","last_undegraded":"2024-08-17T17:46:16.598773+0000","last_fullsized":"2024-08-17T17:46:16.598773+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"435'8193","reported_seq":8227,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.748044+0000","last_change":"2024-08-17T17:44:36.610056+0000","last_active":"2024-08-17T17:46:29.748044+0000","last_peered":"2024-08-17T17:46:29.748044+0000","last_clean":"2024-08-17T17:46:29.748044+0000","last_became_active":"2024-08-17T17:44:36.609357+0000","last_became_peered":"2024-08-17T17:44:36.609357+0000","last_unstale":"2024-08-17T17:46:29.748044+0000","last_undegraded":"2024-08-17T17:46:29.748044+0000","last_fullsized":"2024-08-17T17:46:29.748044+0000","mapping_epoch":410,"log_start":"420'1900","ondisk_log_start":"420'1900","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6293,"log_dups_size":0,"ondisk_log_size":6293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:01.344940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":495124480,"num_objects":7555,"num_object_clones":0,"num_object_copies":22665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8193,"num_write_kb":503936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":329,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.596313+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:46:16.596313+0000","last_peered":"2024-08-17T17:46:16.596313+0000","last_clean":"2024-08-17T17:46:16.596313+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:46:16.596313+0000","last_undegraded":"2024-08-17T17:46:16.596313+0000","last_fullsized":"2024-08-17T17:46:16.596313+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":343,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598958+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:46:16.598958+0000","last_peered":"2024-08-17T17:46:16.598958+0000","last_clean":"2024-08-17T17:46:16.598958+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:46:16.598958+0000","last_undegraded":"2024-08-17T17:46:16.598958+0000","last_fullsized":"2024-08-17T17:46:16.598958+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"435'8121","reported_seq":8155,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.693105+0000","last_change":"2024-08-17T17:44:36.610369+0000","last_active":"2024-08-17T17:46:30.693105+0000","last_peered":"2024-08-17T17:46:30.693105+0000","last_clean":"2024-08-17T17:46:30.693105+0000","last_became_active":"2024-08-17T17:44:36.610096+0000","last_became_peered":"2024-08-17T17:44:36.610096+0000","last_unstale":"2024-08-17T17:46:30.693105+0000","last_undegraded":"2024-08-17T17:46:30.693105+0000","last_fullsized":"2024-08-17T17:46:30.693105+0000","mapping_epoch":410,"log_start":"420'1900","ondisk_log_start":"420'1900","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6221,"log_dups_size":0,"ondisk_log_size":6221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:28.841902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482672640,"num_objects":7365,"num_object_clones":0,"num_object_copies":22095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8121,"num_write_kb":495552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":452,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533288+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:46:16.533288+0000","last_peered":"2024-08-17T17:46:16.533288+0000","last_clean":"2024-08-17T17:46:16.533288+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:46:16.533288+0000","last_undegraded":"2024-08-17T17:46:16.533288+0000","last_fullsized":"2024-08-17T17:46:16.533288+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628590+0000","last_change":"2024-08-17T17:44:45.657126+0000","last_active":"2024-08-17T17:46:29.628590+0000","last_peered":"2024-08-17T17:46:29.628590+0000","last_clean":"2024-08-17T17:46:29.628590+0000","last_became_active":"2024-08-17T17:44:45.656799+0000","last_became_peered":"2024-08-17T17:44:45.656799+0000","last_unstale":"2024-08-17T17:46:29.628590+0000","last_undegraded":"2024-08-17T17:46:29.628590+0000","last_fullsized":"2024-08-17T17:46:29.628590+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:06.941588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":729,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.625517+0000","last_change":"2024-08-17T17:44:45.659396+0000","last_active":"2024-08-17T17:46:29.625517+0000","last_peered":"2024-08-17T17:46:29.625517+0000","last_clean":"2024-08-17T17:46:29.625517+0000","last_became_active":"2024-08-17T17:44:45.659081+0000","last_became_peered":"2024-08-17T17:44:45.659081+0000","last_unstale":"2024-08-17T17:46:29.625517+0000","last_undegraded":"2024-08-17T17:46:29.625517+0000","last_fullsized":"2024-08-17T17:46:29.625517+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:14.819296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"435'8183","reported_seq":8244,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:31.565529+0000","last_change":"2024-08-17T17:44:36.610395+0000","last_active":"2024-08-17T17:46:31.565529+0000","last_peered":"2024-08-17T17:46:31.565529+0000","last_clean":"2024-08-17T17:46:31.565529+0000","last_became_active":"2024-08-17T17:44:36.610060+0000","last_became_peered":"2024-08-17T17:44:36.610060+0000","last_unstale":"2024-08-17T17:46:31.565529+0000","last_undegraded":"2024-08-17T17:46:31.565529+0000","last_fullsized":"2024-08-17T17:46:31.565529+0000","mapping_epoch":410,"log_start":"419'1300","ondisk_log_start":"419'1300","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:23.272119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":488308736,"num_objects":7451,"num_object_clones":0,"num_object_copies":22353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8183,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":776,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.532992+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:46:16.532992+0000","last_peered":"2024-08-17T17:46:16.532992+0000","last_clean":"2024-08-17T17:46:16.532992+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:46:16.532992+0000","last_undegraded":"2024-08-17T17:46:16.532992+0000","last_fullsized":"2024-08-17T17:46:16.532992+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1029,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533385+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:46:16.533385+0000","last_peered":"2024-08-17T17:46:16.533385+0000","last_clean":"2024-08-17T17:46:16.533385+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:46:16.533385+0000","last_undegraded":"2024-08-17T17:46:16.533385+0000","last_fullsized":"2024-08-17T17:46:16.533385+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":362,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533418+0000","last_change":"2024-08-17T17:44:45.651175+0000","last_active":"2024-08-17T17:46:16.533418+0000","last_peered":"2024-08-17T17:46:16.533418+0000","last_clean":"2024-08-17T17:46:16.533418+0000","last_became_active":"2024-08-17T17:44:45.650847+0000","last_became_peered":"2024-08-17T17:44:45.650847+0000","last_unstale":"2024-08-17T17:46:16.533418+0000","last_undegraded":"2024-08-17T17:46:16.533418+0000","last_fullsized":"2024-08-17T17:46:16.533418+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:30.892351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":343,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:26.058614+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:46:26.058614+0000","last_peered":"2024-08-17T17:46:26.058614+0000","last_clean":"2024-08-17T17:46:26.058614+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:46:26.058614+0000","last_undegraded":"2024-08-17T17:46:26.058614+0000","last_fullsized":"2024-08-17T17:46:26.058614+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"435'8088","reported_seq":8147,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:31.456572+0000","last_change":"2024-08-17T17:44:36.968089+0000","last_active":"2024-08-17T17:46:31.456572+0000","last_peered":"2024-08-17T17:46:31.456572+0000","last_clean":"2024-08-17T17:46:31.456572+0000","last_became_active":"2024-08-17T17:44:36.967095+0000","last_became_peered":"2024-08-17T17:44:36.967095+0000","last_unstale":"2024-08-17T17:46:31.456572+0000","last_undegraded":"2024-08-17T17:46:31.456572+0000","last_fullsized":"2024-08-17T17:46:31.456572+0000","mapping_epoch":410,"log_start":"418'700","ondisk_log_start":"418'700","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:57.468862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480903168,"num_objects":7338,"num_object_clones":0,"num_object_copies":22014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8088,"num_write_kb":493632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":535,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629188+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:46:29.629188+0000","last_peered":"2024-08-17T17:46:29.629188+0000","last_clean":"2024-08-17T17:46:29.629188+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:46:29.629188+0000","last_undegraded":"2024-08-17T17:46:29.629188+0000","last_fullsized":"2024-08-17T17:46:29.629188+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":816,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.532906+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:46:16.532906+0000","last_peered":"2024-08-17T17:46:16.532906+0000","last_clean":"2024-08-17T17:46:16.532906+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:46:16.532906+0000","last_undegraded":"2024-08-17T17:46:16.532906+0000","last_fullsized":"2024-08-17T17:46:16.532906+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":747,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.635399+0000","last_change":"2024-08-17T17:44:45.659724+0000","last_active":"2024-08-17T17:46:29.635399+0000","last_peered":"2024-08-17T17:46:29.635399+0000","last_clean":"2024-08-17T17:46:29.635399+0000","last_became_active":"2024-08-17T17:44:45.654153+0000","last_became_peered":"2024-08-17T17:44:45.654153+0000","last_unstale":"2024-08-17T17:46:29.635399+0000","last_undegraded":"2024-08-17T17:46:29.635399+0000","last_fullsized":"2024-08-17T17:46:29.635399+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.165904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":736,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630612+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:46:29.630612+0000","last_peered":"2024-08-17T17:46:29.630612+0000","last_clean":"2024-08-17T17:46:29.630612+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:46:29.630612+0000","last_undegraded":"2024-08-17T17:46:29.630612+0000","last_fullsized":"2024-08-17T17:46:29.630612+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"435'8226","reported_seq":8260,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.462084+0000","last_change":"2024-08-17T17:44:36.966264+0000","last_active":"2024-08-17T17:46:30.462084+0000","last_peered":"2024-08-17T17:46:30.462084+0000","last_clean":"2024-08-17T17:46:30.462084+0000","last_became_active":"2024-08-17T17:44:36.965940+0000","last_became_peered":"2024-08-17T17:44:36.965940+0000","last_unstale":"2024-08-17T17:46:30.462084+0000","last_undegraded":"2024-08-17T17:46:30.462084+0000","last_fullsized":"2024-08-17T17:46:30.462084+0000","mapping_epoch":410,"log_start":"418'100","ondisk_log_start":"418'100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:16.157942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486670336,"num_objects":7426,"num_object_clones":0,"num_object_copies":22278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8226,"num_write_kb":500864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":501,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.532928+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:46:16.532928+0000","last_peered":"2024-08-17T17:46:16.532928+0000","last_clean":"2024-08-17T17:46:16.532928+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:46:16.532928+0000","last_undegraded":"2024-08-17T17:46:16.532928+0000","last_fullsized":"2024-08-17T17:46:16.532928+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":482,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.635534+0000","last_change":"2024-08-17T17:44:45.679401+0000","last_active":"2024-08-17T17:46:29.635534+0000","last_peered":"2024-08-17T17:46:29.635534+0000","last_clean":"2024-08-17T17:46:29.635534+0000","last_became_active":"2024-08-17T17:44:45.664192+0000","last_became_peered":"2024-08-17T17:44:45.664192+0000","last_unstale":"2024-08-17T17:46:29.635534+0000","last_undegraded":"2024-08-17T17:46:29.635534+0000","last_fullsized":"2024-08-17T17:46:29.635534+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:21.066280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":729,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628680+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:46:29.628680+0000","last_peered":"2024-08-17T17:46:29.628680+0000","last_clean":"2024-08-17T17:46:29.628680+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:46:29.628680+0000","last_undegraded":"2024-08-17T17:46:29.628680+0000","last_fullsized":"2024-08-17T17:46:29.628680+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":776,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628244+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:46:29.628244+0000","last_peered":"2024-08-17T17:46:29.628244+0000","last_clean":"2024-08-17T17:46:29.628244+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:46:29.628244+0000","last_undegraded":"2024-08-17T17:46:29.628244+0000","last_fullsized":"2024-08-17T17:46:29.628244+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"435'8230","reported_seq":8264,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.829618+0000","last_change":"2024-08-17T17:44:36.610536+0000","last_active":"2024-08-17T17:46:29.829618+0000","last_peered":"2024-08-17T17:46:29.829618+0000","last_clean":"2024-08-17T17:46:29.829618+0000","last_became_active":"2024-08-17T17:44:36.610195+0000","last_became_peered":"2024-08-17T17:44:36.610195+0000","last_unstale":"2024-08-17T17:46:29.829618+0000","last_undegraded":"2024-08-17T17:46:29.829618+0000","last_fullsized":"2024-08-17T17:46:29.829618+0000","mapping_epoch":410,"log_start":"420'1900","ondisk_log_start":"420'1900","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6330,"log_dups_size":0,"ondisk_log_size":6330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:16.686185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":498466816,"num_objects":7606,"num_object_clones":0,"num_object_copies":22818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":506752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":452,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533677+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:46:16.533677+0000","last_peered":"2024-08-17T17:46:16.533677+0000","last_clean":"2024-08-17T17:46:16.533677+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:46:16.533677+0000","last_undegraded":"2024-08-17T17:46:16.533677+0000","last_fullsized":"2024-08-17T17:46:16.533677+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1152,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533549+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:46:16.533549+0000","last_peered":"2024-08-17T17:46:16.533549+0000","last_clean":"2024-08-17T17:46:16.533549+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:46:16.533549+0000","last_undegraded":"2024-08-17T17:46:16.533549+0000","last_fullsized":"2024-08-17T17:46:16.533549+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":554,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.624839+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:46:29.624839+0000","last_peered":"2024-08-17T17:46:29.624839+0000","last_clean":"2024-08-17T17:46:29.624839+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:46:29.624839+0000","last_undegraded":"2024-08-17T17:46:29.624839+0000","last_fullsized":"2024-08-17T17:46:29.624839+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":92,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533505+0000","last_change":"2024-08-17T17:44:45.659123+0000","last_active":"2024-08-17T17:46:16.533505+0000","last_peered":"2024-08-17T17:46:16.533505+0000","last_clean":"2024-08-17T17:46:16.533505+0000","last_became_active":"2024-08-17T17:44:45.658785+0000","last_became_peered":"2024-08-17T17:44:45.658785+0000","last_unstale":"2024-08-17T17:46:16.533505+0000","last_undegraded":"2024-08-17T17:46:16.533505+0000","last_fullsized":"2024-08-17T17:46:16.533505+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:30.466001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"435'8239","reported_seq":8273,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.677118+0000","last_change":"2024-08-17T17:44:36.607785+0000","last_active":"2024-08-17T17:46:30.677118+0000","last_peered":"2024-08-17T17:46:30.677118+0000","last_clean":"2024-08-17T17:46:30.677118+0000","last_became_active":"2024-08-17T17:44:36.607472+0000","last_became_peered":"2024-08-17T17:44:36.607472+0000","last_unstale":"2024-08-17T17:46:30.677118+0000","last_undegraded":"2024-08-17T17:46:30.677118+0000","last_fullsized":"2024-08-17T17:46:30.677118+0000","mapping_epoch":410,"log_start":"418'700","ondisk_log_start":"418'700","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7539,"log_dups_size":0,"ondisk_log_size":7539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:39.734707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":489357312,"num_objects":7467,"num_object_clones":0,"num_object_copies":22401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8239,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":746,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534181+0000","last_change":"2024-08-17T17:44:52.091382+0000","last_active":"2024-08-17T17:46:16.534181+0000","last_peered":"2024-08-17T17:46:16.534181+0000","last_clean":"2024-08-17T17:46:16.534181+0000","last_became_active":"2024-08-17T17:44:48.663244+0000","last_became_peered":"2024-08-17T17:44:48.663244+0000","last_unstale":"2024-08-17T17:46:16.534181+0000","last_undegraded":"2024-08-17T17:46:16.534181+0000","last_fullsized":"2024-08-17T17:46:16.534181+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:44:52.091249+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:18.054213+0000","scrub_duration":0.001820239,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533658+0000","last_change":"2024-08-17T17:44:45.653860+0000","last_active":"2024-08-17T17:46:16.533658+0000","last_peered":"2024-08-17T17:46:16.533658+0000","last_clean":"2024-08-17T17:46:16.533658+0000","last_became_active":"2024-08-17T17:44:45.652151+0000","last_became_peered":"2024-08-17T17:44:45.652151+0000","last_unstale":"2024-08-17T17:46:16.533658+0000","last_undegraded":"2024-08-17T17:46:16.533658+0000","last_fullsized":"2024-08-17T17:46:16.533658+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:17.052902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":1021,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629244+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:46:29.629244+0000","last_peered":"2024-08-17T17:46:29.629244+0000","last_clean":"2024-08-17T17:46:29.629244+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:46:29.629244+0000","last_undegraded":"2024-08-17T17:46:29.629244+0000","last_fullsized":"2024-08-17T17:46:29.629244+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":745,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629549+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:46:29.629549+0000","last_peered":"2024-08-17T17:46:29.629549+0000","last_clean":"2024-08-17T17:46:29.629549+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:46:29.629549+0000","last_undegraded":"2024-08-17T17:46:29.629549+0000","last_fullsized":"2024-08-17T17:46:29.629549+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"435'8113","reported_seq":8148,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.672227+0000","last_change":"2024-08-17T17:44:36.611239+0000","last_active":"2024-08-17T17:46:30.672227+0000","last_peered":"2024-08-17T17:46:30.672227+0000","last_clean":"2024-08-17T17:46:30.672227+0000","last_became_active":"2024-08-17T17:44:36.610907+0000","last_became_peered":"2024-08-17T17:44:36.610907+0000","last_unstale":"2024-08-17T17:46:30.672227+0000","last_undegraded":"2024-08-17T17:46:30.672227+0000","last_fullsized":"2024-08-17T17:46:30.672227+0000","mapping_epoch":410,"log_start":"420'1900","ondisk_log_start":"420'1900","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6213,"log_dups_size":0,"ondisk_log_size":6213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:43.474878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486146072,"num_objects":7419,"num_object_clones":0,"num_object_copies":22257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7419,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8113,"num_write_kb":496961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":548,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630258+0000","last_change":"2024-08-17T17:44:48.665077+0000","last_active":"2024-08-17T17:46:29.630258+0000","last_peered":"2024-08-17T17:46:29.630258+0000","last_clean":"2024-08-17T17:46:29.630258+0000","last_became_active":"2024-08-17T17:44:48.664818+0000","last_became_peered":"2024-08-17T17:44:48.664818+0000","last_unstale":"2024-08-17T17:46:29.630258+0000","last_undegraded":"2024-08-17T17:46:29.630258+0000","last_fullsized":"2024-08-17T17:46:29.630258+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:36.058763+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":742,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629151+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:46:29.629151+0000","last_peered":"2024-08-17T17:46:29.629151+0000","last_clean":"2024-08-17T17:46:29.629151+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:46:29.629151+0000","last_undegraded":"2024-08-17T17:46:29.629151+0000","last_fullsized":"2024-08-17T17:46:29.629151+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1006,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.037695+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:46:21.037695+0000","last_peered":"2024-08-17T17:46:21.037695+0000","last_clean":"2024-08-17T17:46:21.037695+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:46:21.037695+0000","last_undegraded":"2024-08-17T17:46:21.037695+0000","last_fullsized":"2024-08-17T17:46:21.037695+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":740,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630647+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:46:29.630647+0000","last_peered":"2024-08-17T17:46:29.630647+0000","last_clean":"2024-08-17T17:46:29.630647+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:46:29.630647+0000","last_undegraded":"2024-08-17T17:46:29.630647+0000","last_fullsized":"2024-08-17T17:46:29.630647+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"435'8225","reported_seq":8259,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:30.677850+0000","last_change":"2024-08-17T17:44:36.966926+0000","last_active":"2024-08-17T17:46:30.677850+0000","last_peered":"2024-08-17T17:46:30.677850+0000","last_clean":"2024-08-17T17:46:30.677850+0000","last_became_active":"2024-08-17T17:44:36.966608+0000","last_became_peered":"2024-08-17T17:44:36.966608+0000","last_unstale":"2024-08-17T17:46:30.677850+0000","last_undegraded":"2024-08-17T17:46:30.677850+0000","last_fullsized":"2024-08-17T17:46:30.677850+0000","mapping_epoch":410,"log_start":"420'2100","ondisk_log_start":"420'2100","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6125,"log_dups_size":0,"ondisk_log_size":6125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:39.384622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491323392,"num_objects":7497,"num_object_clones":0,"num_object_copies":22491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8225,"num_write_kb":503104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":453,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598601+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:46:16.598601+0000","last_peered":"2024-08-17T17:46:16.598601+0000","last_clean":"2024-08-17T17:46:16.598601+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:46:16.598601+0000","last_undegraded":"2024-08-17T17:46:16.598601+0000","last_fullsized":"2024-08-17T17:46:16.598601+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":57,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534318+0000","last_change":"2024-08-17T17:44:45.670035+0000","last_active":"2024-08-17T17:46:16.534318+0000","last_peered":"2024-08-17T17:46:16.534318+0000","last_clean":"2024-08-17T17:46:16.534318+0000","last_became_active":"2024-08-17T17:44:45.655566+0000","last_became_peered":"2024-08-17T17:44:45.655566+0000","last_unstale":"2024-08-17T17:46:16.534318+0000","last_undegraded":"2024-08-17T17:46:16.534318+0000","last_fullsized":"2024-08-17T17:46:16.534318+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:17:32.230235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1252,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628585+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:46:29.628585+0000","last_peered":"2024-08-17T17:46:29.628585+0000","last_clean":"2024-08-17T17:46:29.628585+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:46:29.628585+0000","last_undegraded":"2024-08-17T17:46:29.628585+0000","last_fullsized":"2024-08-17T17:46:29.628585+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.532756+0000","last_change":"2024-08-17T17:44:45.656210+0000","last_active":"2024-08-17T17:46:16.532756+0000","last_peered":"2024-08-17T17:46:16.532756+0000","last_clean":"2024-08-17T17:46:16.532756+0000","last_became_active":"2024-08-17T17:44:45.655853+0000","last_became_peered":"2024-08-17T17:44:45.655853+0000","last_unstale":"2024-08-17T17:46:16.532756+0000","last_undegraded":"2024-08-17T17:46:16.532756+0000","last_fullsized":"2024-08-17T17:46:16.532756+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:01:57.057139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"435'8153","reported_seq":8196,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.829600+0000","last_change":"2024-08-17T17:44:45.659316+0000","last_active":"2024-08-17T17:46:29.829600+0000","last_peered":"2024-08-17T17:46:29.829600+0000","last_clean":"2024-08-17T17:46:29.829600+0000","last_became_active":"2024-08-17T17:44:45.658966+0000","last_became_peered":"2024-08-17T17:44:45.658966+0000","last_unstale":"2024-08-17T17:46:29.829600+0000","last_undegraded":"2024-08-17T17:46:29.829600+0000","last_fullsized":"2024-08-17T17:46:29.829600+0000","mapping_epoch":413,"log_start":"420'1900","ondisk_log_start":"420'1900","created":410,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:22.822524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":490143744,"num_objects":7479,"num_object_clones":0,"num_object_copies":22437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8153,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":421,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533924+0000","last_change":"2024-08-17T17:44:51.112004+0000","last_active":"2024-08-17T17:46:16.533924+0000","last_peered":"2024-08-17T17:46:16.533924+0000","last_clean":"2024-08-17T17:46:16.533924+0000","last_became_active":"2024-08-17T17:44:48.661437+0000","last_became_peered":"2024-08-17T17:44:48.661437+0000","last_unstale":"2024-08-17T17:46:16.533924+0000","last_undegraded":"2024-08-17T17:46:16.533924+0000","last_fullsized":"2024-08-17T17:46:16.533924+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:00.448235+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":751,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630357+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:46:29.630357+0000","last_peered":"2024-08-17T17:46:29.630357+0000","last_clean":"2024-08-17T17:46:29.630357+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:46:29.630357+0000","last_undegraded":"2024-08-17T17:46:29.630357+0000","last_fullsized":"2024-08-17T17:46:29.630357+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":69,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534174+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:46:16.534174+0000","last_peered":"2024-08-17T17:46:16.534174+0000","last_clean":"2024-08-17T17:46:16.534174+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:46:16.534174+0000","last_undegraded":"2024-08-17T17:46:16.534174+0000","last_fullsized":"2024-08-17T17:46:16.534174+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":380,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.596524+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:46:16.596524+0000","last_peered":"2024-08-17T17:46:16.596524+0000","last_clean":"2024-08-17T17:46:16.596524+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:46:16.596524+0000","last_undegraded":"2024-08-17T17:46:16.596524+0000","last_fullsized":"2024-08-17T17:46:16.596524+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":343,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598842+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:46:16.598842+0000","last_peered":"2024-08-17T17:46:16.598842+0000","last_clean":"2024-08-17T17:46:16.598842+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:46:16.598842+0000","last_undegraded":"2024-08-17T17:46:16.598842+0000","last_fullsized":"2024-08-17T17:46:16.598842+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":353,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.599050+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:46:16.599050+0000","last_peered":"2024-08-17T17:46:16.599050+0000","last_clean":"2024-08-17T17:46:16.599050+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:46:16.599050+0000","last_undegraded":"2024-08-17T17:46:16.599050+0000","last_fullsized":"2024-08-17T17:46:16.599050+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":727,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629443+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:46:29.629443+0000","last_peered":"2024-08-17T17:46:29.629443+0000","last_clean":"2024-08-17T17:46:29.629443+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:46:29.629443+0000","last_undegraded":"2024-08-17T17:46:29.629443+0000","last_fullsized":"2024-08-17T17:46:29.629443+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":452,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534142+0000","last_change":"2024-08-17T17:44:45.668086+0000","last_active":"2024-08-17T17:46:16.534142+0000","last_peered":"2024-08-17T17:46:16.534142+0000","last_clean":"2024-08-17T17:46:16.534142+0000","last_became_active":"2024-08-17T17:44:45.654205+0000","last_became_peered":"2024-08-17T17:44:45.654205+0000","last_unstale":"2024-08-17T17:46:16.534142+0000","last_undegraded":"2024-08-17T17:46:16.534142+0000","last_fullsized":"2024-08-17T17:46:16.534142+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:30.323982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533645+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:46:16.533645+0000","last_peered":"2024-08-17T17:46:16.533645+0000","last_clean":"2024-08-17T17:46:16.533645+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:46:16.533645+0000","last_undegraded":"2024-08-17T17:46:16.533645+0000","last_fullsized":"2024-08-17T17:46:16.533645+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":337,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533043+0000","last_change":"2024-08-17T17:44:45.652198+0000","last_active":"2024-08-17T17:46:16.533043+0000","last_peered":"2024-08-17T17:46:16.533043+0000","last_clean":"2024-08-17T17:46:16.533043+0000","last_became_active":"2024-08-17T17:44:45.651854+0000","last_became_peered":"2024-08-17T17:44:45.651854+0000","last_unstale":"2024-08-17T17:46:16.533043+0000","last_undegraded":"2024-08-17T17:46:16.533043+0000","last_fullsized":"2024-08-17T17:46:16.533043+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:49:58.681636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":760,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630074+0000","last_change":"2024-08-17T17:44:45.651808+0000","last_active":"2024-08-17T17:46:29.630074+0000","last_peered":"2024-08-17T17:46:29.630074+0000","last_clean":"2024-08-17T17:46:29.630074+0000","last_became_active":"2024-08-17T17:44:45.651478+0000","last_became_peered":"2024-08-17T17:44:45.651478+0000","last_unstale":"2024-08-17T17:46:29.630074+0000","last_undegraded":"2024-08-17T17:46:29.630074+0000","last_fullsized":"2024-08-17T17:46:29.630074+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:04.589247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":408,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628394+0000","last_change":"2024-08-17T17:44:45.661728+0000","last_active":"2024-08-17T17:46:29.628394+0000","last_peered":"2024-08-17T17:46:29.628394+0000","last_clean":"2024-08-17T17:46:29.628394+0000","last_became_active":"2024-08-17T17:44:45.661337+0000","last_became_peered":"2024-08-17T17:44:45.661337+0000","last_unstale":"2024-08-17T17:46:29.628394+0000","last_undegraded":"2024-08-17T17:46:29.628394+0000","last_fullsized":"2024-08-17T17:46:29.628394+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:54:21.723874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":231,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533793+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:46:16.533793+0000","last_peered":"2024-08-17T17:46:16.533793+0000","last_clean":"2024-08-17T17:46:16.533793+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:46:16.533793+0000","last_undegraded":"2024-08-17T17:46:16.533793+0000","last_fullsized":"2024-08-17T17:46:16.533793+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.635654+0000","last_change":"2024-08-17T17:44:45.663565+0000","last_active":"2024-08-17T17:46:29.635654+0000","last_peered":"2024-08-17T17:46:29.635654+0000","last_clean":"2024-08-17T17:46:29.635654+0000","last_became_active":"2024-08-17T17:44:45.663237+0000","last_became_peered":"2024-08-17T17:44:45.663237+0000","last_unstale":"2024-08-17T17:46:29.635654+0000","last_undegraded":"2024-08-17T17:46:29.635654+0000","last_fullsized":"2024-08-17T17:46:29.635654+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:26:34.711189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":733,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630534+0000","last_change":"2024-08-17T17:44:45.662891+0000","last_active":"2024-08-17T17:46:29.630534+0000","last_peered":"2024-08-17T17:46:29.630534+0000","last_clean":"2024-08-17T17:46:29.630534+0000","last_became_active":"2024-08-17T17:44:45.662580+0000","last_became_peered":"2024-08-17T17:44:45.662580+0000","last_unstale":"2024-08-17T17:46:29.630534+0000","last_undegraded":"2024-08-17T17:46:29.630534+0000","last_fullsized":"2024-08-17T17:46:29.630534+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:48.627337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":736,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628113+0000","last_change":"2024-08-17T17:44:45.657935+0000","last_active":"2024-08-17T17:46:29.628113+0000","last_peered":"2024-08-17T17:46:29.628113+0000","last_clean":"2024-08-17T17:46:29.628113+0000","last_became_active":"2024-08-17T17:44:45.657525+0000","last_became_peered":"2024-08-17T17:44:45.657525+0000","last_unstale":"2024-08-17T17:46:29.628113+0000","last_undegraded":"2024-08-17T17:46:29.628113+0000","last_fullsized":"2024-08-17T17:46:29.628113+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:55.691041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":167,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628203+0000","last_change":"2024-08-17T17:44:45.661307+0000","last_active":"2024-08-17T17:46:29.628203+0000","last_peered":"2024-08-17T17:46:29.628203+0000","last_clean":"2024-08-17T17:46:29.628203+0000","last_became_active":"2024-08-17T17:44:45.660976+0000","last_became_peered":"2024-08-17T17:44:45.660976+0000","last_unstale":"2024-08-17T17:46:29.628203+0000","last_undegraded":"2024-08-17T17:46:29.628203+0000","last_fullsized":"2024-08-17T17:46:29.628203+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:22.725842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":327,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598477+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:46:16.598477+0000","last_peered":"2024-08-17T17:46:16.598477+0000","last_clean":"2024-08-17T17:46:16.598477+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:46:16.598477+0000","last_undegraded":"2024-08-17T17:46:16.598477+0000","last_fullsized":"2024-08-17T17:46:16.598477+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":264,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.629412+0000","last_change":"2024-08-17T17:44:45.660322+0000","last_active":"2024-08-17T17:46:29.629412+0000","last_peered":"2024-08-17T17:46:29.629412+0000","last_clean":"2024-08-17T17:46:29.629412+0000","last_became_active":"2024-08-17T17:44:45.660018+0000","last_became_peered":"2024-08-17T17:44:45.660018+0000","last_unstale":"2024-08-17T17:46:29.629412+0000","last_undegraded":"2024-08-17T17:46:29.629412+0000","last_fullsized":"2024-08-17T17:46:29.629412+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:57.732705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":120,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630397+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:46:29.630397+0000","last_peered":"2024-08-17T17:46:29.630397+0000","last_clean":"2024-08-17T17:46:29.630397+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:46:29.630397+0000","last_undegraded":"2024-08-17T17:46:29.630397+0000","last_fullsized":"2024-08-17T17:46:29.630397+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":336,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598759+0000","last_change":"2024-08-17T17:44:45.657401+0000","last_active":"2024-08-17T17:46:16.598759+0000","last_peered":"2024-08-17T17:46:16.598759+0000","last_clean":"2024-08-17T17:46:16.598759+0000","last_became_active":"2024-08-17T17:44:45.657074+0000","last_became_peered":"2024-08-17T17:44:45.657074+0000","last_unstale":"2024-08-17T17:46:16.598759+0000","last_undegraded":"2024-08-17T17:46:16.598759+0000","last_fullsized":"2024-08-17T17:46:16.598759+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:19.924661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":224,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.534009+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:46:16.534009+0000","last_peered":"2024-08-17T17:46:16.534009+0000","last_clean":"2024-08-17T17:46:16.534009+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:46:16.534009+0000","last_undegraded":"2024-08-17T17:46:16.534009+0000","last_fullsized":"2024-08-17T17:46:16.534009+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":336,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.598395+0000","last_change":"2024-08-17T17:44:45.657702+0000","last_active":"2024-08-17T17:46:16.598395+0000","last_peered":"2024-08-17T17:46:16.598395+0000","last_clean":"2024-08-17T17:46:16.598395+0000","last_became_active":"2024-08-17T17:44:45.657367+0000","last_became_peered":"2024-08-17T17:44:45.657367+0000","last_unstale":"2024-08-17T17:46:16.598395+0000","last_undegraded":"2024-08-17T17:46:16.598395+0000","last_fullsized":"2024-08-17T17:46:16.598395+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:29.447263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":234,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.533737+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:46:16.533737+0000","last_peered":"2024-08-17T17:46:16.533737+0000","last_clean":"2024-08-17T17:46:16.533737+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:46:16.533737+0000","last_undegraded":"2024-08-17T17:46:16.533737+0000","last_fullsized":"2024-08-17T17:46:16.533737+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.635449+0000","last_change":"2024-08-17T17:44:45.660616+0000","last_active":"2024-08-17T17:46:29.635449+0000","last_peered":"2024-08-17T17:46:29.635449+0000","last_clean":"2024-08-17T17:46:29.635449+0000","last_became_active":"2024-08-17T17:44:45.660198+0000","last_became_peered":"2024-08-17T17:44:45.660198+0000","last_unstale":"2024-08-17T17:46:29.635449+0000","last_undegraded":"2024-08-17T17:46:29.635449+0000","last_fullsized":"2024-08-17T17:46:29.635449+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:06.005702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":54,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.628242+0000","last_change":"2024-08-17T17:44:45.662070+0000","last_active":"2024-08-17T17:46:29.628242+0000","last_peered":"2024-08-17T17:46:29.628242+0000","last_clean":"2024-08-17T17:46:29.628242+0000","last_became_active":"2024-08-17T17:44:45.661702+0000","last_became_peered":"2024-08-17T17:44:45.661702+0000","last_unstale":"2024-08-17T17:46:29.628242+0000","last_undegraded":"2024-08-17T17:46:29.628242+0000","last_fullsized":"2024-08-17T17:46:29.628242+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:11.915663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":342,"reported_epoch":434,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.596253+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:46:16.596253+0000","last_peered":"2024-08-17T17:46:16.596253+0000","last_clean":"2024-08-17T17:46:16.596253+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:46:16.596253+0000","last_undegraded":"2024-08-17T17:46:16.596253+0000","last_fullsized":"2024-08-17T17:46:16.596253+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":771,"reported_epoch":435,"state":"active+clean","last_fresh":"2024-08-17T17:46:29.630021+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:46:29.630021+0000","last_peered":"2024-08-17T17:46:29.630021+0000","last_clean":"2024-08-17T17:46:29.630021+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:46:29.630021+0000","last_undegraded":"2024-08-17T17:46:29.630021+0000","last_fullsized":"2024-08-17T17:46:29.630021+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":7808417816,"num_objects":119148,"num_object_clones":0,"num_object_copies":357444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":119148,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130810,"num_write_kb":7998593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":23407833088,"data_stored":23407820872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113310,"ondisk_log_size":113310,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201796,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3311632,"kb_used_data":2387908,"kb_used_omap":1,"kb_used_meta":923646,"kb_avail":90433520,"statfs":{"total":95995035648,"available":92603924480,"internally_reserved":0,"allocated":2445217792,"data_stored":2443099818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":945813963},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922292,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3861304,"kb_used_data":3372528,"kb_used_omap":7,"kb_used_meta":488760,"kb_avail":89883848,"statfs":{"total":95995035648,"available":92041060352,"internally_reserved":0,"allocated":3453468672,"data_stored":3451362365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":500490934},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085816,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2993796,"kb_used_data":2401336,"kb_used_omap":7,"kb_used_meta":592376,"kb_avail":90751356,"statfs":{"total":95995035648,"available":92929388544,"internally_reserved":0,"allocated":2458968064,"data_stored":2456858621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":606593465},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986706,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5658060,"kb_used_data":4782660,"kb_used_omap":1,"kb_used_meta":875262,"kb_avail":88087092,"statfs":{"total":95995035648,"available":90201182208,"internally_reserved":0,"allocated":4897443840,"data_stored":4895325828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":896268742},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":0,"seq":468151435597,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599124,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4289080,"kb_used_data":3328624,"kb_used_omap":7,"kb_used_meta":960440,"kb_avail":89456072,"statfs":{"total":95995035648,"available":91603017728,"internally_reserved":0,"allocated":3408510976,"data_stored":3406408513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":983490733},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762650,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3017452,"kb_used_data":2165284,"kb_used_omap":7,"kb_used_meta":852152,"kb_avail":90727700,"statfs":{"total":95995035648,"available":92905164800,"internally_reserved":0,"allocated":2217250816,"data_stored":2215159227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":872604065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926176,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5738756,"kb_used_data":4768572,"kb_used_omap":7,"kb_used_meta":970168,"kb_avail":88006396,"statfs":{"total":95995035648,"available":90118549504,"internally_reserved":0,"allocated":4883017728,"data_stored":4880903985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":993452636},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4872605696,"data_stored":4872601624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1950613504,"data_stored":1950613504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3390504960,"data_stored":3390504960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4882698240,"data_stored":4882694168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2440560640,"data_stored":2440560640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3440250880,"data_stored":3440246808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2430599168,"data_stored":2430599168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:46:38.472 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:46:38.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-08-17T17:46:38.474 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:38.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:38.612 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:38.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:38 smithi064 ceph-mon[104516]: pgmap v1404: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 925 op/s 2024-08-17T17:46:38.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:38 smithi064 ceph-mon[109246]: pgmap v1404: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 925 op/s 2024-08-17T17:46:38.714 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:38.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:38.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:38.892 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:38 smithi175 ceph-mon[101434]: pgmap v1404: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 925 op/s 2024-08-17T17:46:38.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:46:38.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:39.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:39.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:39.168 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:39.270 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:39.315 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:39.417 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:39.453 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:39.461 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:39.555 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:39.587 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:39.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:39.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:39.722 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:39.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:39.852 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:39.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:39.986 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:40.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:40.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:40.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:40.266 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:40.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.410 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:40.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:40.551 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:40.653 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:40.691 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:40.727 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:40.728 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:40.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:46:40.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:46:40.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:46:40.729 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:40.729 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:40.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.828 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:40 smithi175 ceph-mon[101434]: pgmap v1405: 99 pgs: 99 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:46:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4099449926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:40.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[104516]: pgmap v1405: 99 pgs: 99 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:46:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:40.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4099449926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[109246]: pgmap v1405: 99 pgs: 99 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:46:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:40.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4099449926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.962 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:40.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:40.991 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:41.063 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:41.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:41.200 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:41.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:41.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:41.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:41.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:41.506 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:41.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:41.656 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:41.758 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:41.800 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:41.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:41.941 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:42.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:42.078 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:42.180 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:42.219 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:42.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:42.350 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:42.358 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:42.459 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.498 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:42.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:42.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:42.633 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:42.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:42.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:42.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:42 smithi064 ceph-mon[104516]: pgmap v1406: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:46:42.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:42 smithi064 ceph-mon[109246]: pgmap v1406: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:46:42.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:42.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:42.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:42 smithi175 ceph-mon[101434]: pgmap v1406: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:46:43.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:43.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:43.141 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:43.186 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:43.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:43.325 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:43.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:43.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:43.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:43.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.598 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:43.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:43.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:46:43.698 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:43.734 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:43.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:43.874 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:43.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:43.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:44.017 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:44.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:44.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:44.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:43] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-08-17T17:46:44.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:44.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:44.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:44.425 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:44.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.563 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:44.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:44.664 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:44.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:44.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:44 smithi064 ceph-mon[104516]: pgmap v1407: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2024-08-17T17:46:44.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:44 smithi064 ceph-mon[109246]: pgmap v1407: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2024-08-17T17:46:44.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:44.837 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:44 smithi175 ceph-mon[101434]: pgmap v1407: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2024-08-17T17:46:44.939 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:44.981 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:44.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:45.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:45.116 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:45.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:45.267 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:45.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:221001 8.21 8:8407315b:::benchmark_data_smithi064_242541_object98508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 474.88944320600001, 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.682936939, 2024-08-17T17:46:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 221001 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732143+0000", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4499999999999997e-07 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732147+0000", 2024-08-17T17:46:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.269e-06 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732156+0000", 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0149999999999992e-06 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732160+0000", 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3449999999999999e-06 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732206+0000", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6066000000000002e-05 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:49.732213+0000", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9280000000000004e-06 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351505+0000", 2024-08-17T17:46:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61929169299999998 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:50.351518+0000", 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2826000000000001e-05 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409384+0000", 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057866347 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409504+0000", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119895 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.409764+0000", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026007200000000003 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.411743+0000", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019789 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414636+0000", 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002893161 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414860+0000", 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000223816 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414939+0000", 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8861000000000003e-05 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.414991+0000", 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2759000000000001e-05 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:38:51.415080+0000", 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8341e-05 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124576 9.5 9:a86362b4:::benchmark_data_smithi064_275711_object124575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 314.063315837, 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2467486480000001, 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124576 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558270+0000", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558271+0000", 2024-08-17T17:46:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.269e-06 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558291+0000", 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9082000000000001e-05 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558306+0000", 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5187999999999999e-05 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558311+0000", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2889999999999997e-06 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558781+0000", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047030999999999999 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.558984+0000", 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020240899999999999 2024-08-17T17:46:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.559149+0000", 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016526299999999999 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.563915+0000", 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047659030000000002 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.569779+0000", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058636549999999997 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601496+0000", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031717707999999997 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601503+0000", 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0729999999999999e-06 2024-08-17T17:46:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601309+0000", 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99980544400000004 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601326+0000", 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7337999999999999e-05 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.804957+0000", 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2036309160000001 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.805019+0000", 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1801e-05 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124594 9.5 9:a480dbe2:::benchmark_data_smithi064_275711_object124593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 314.04582228499999, 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2331379170000001, 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124594 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575764+0000", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575765+0000", 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0690000000000001e-06 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575786+0000", 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1311999999999999e-05 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575798+0000", 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1841000000000001e-05 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575806+0000", 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9170000000000006e-06 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.575873+0000", 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6804999999999995e-05 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.576025+0000", 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015275299999999999 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.576172+0000", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014673099999999999 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.581021+0000", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048487820000000003 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.586844+0000", 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058226750000000002 2024-08-17T17:46:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601612+0000", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014768325000000001 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.601623+0000", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0984e-05 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601380+0000", 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.999756914 2024-08-17T17:46:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601391+0000", 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1615e-05 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808792+0000", 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.207400504 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808902+0000", 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010969 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124652 9.5 9:aa47bb3e:::benchmark_data_smithi064_275711_object124651:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.97134923599998, 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1929173679999998, 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124652 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650237+0000", 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650238+0000", 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7e-06 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650263+0000", 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4824000000000002e-05 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650279+0000", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6161999999999999e-05 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650288+0000", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1540000000000002e-06 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650364+0000", 2024-08-17T17:46:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6188999999999998e-05 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.650376+0000", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2605999999999999e-05 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601433+0000", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95105666799999999 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601441+0000", 2024-08-17T17:46:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8639999999999996e-06 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.808956+0000", 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2075150640000001 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.809085+0000", 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012875399999999999 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.809245+0000", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016023799999999999 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.819892+0000", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010647301 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822932+0000", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030402010000000002 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837245+0000", 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014312147000000001 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837310+0000", 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5772999999999995e-05 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843154+0000", 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058437230000000003 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124655 9.5 9:a81f9d47:::benchmark_data_smithi064_275711_object124654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.96918010500002, 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1968849239999999, 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124655 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652406+0000", 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652407+0000", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018e-06 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652436+0000", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.934e-05 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652447+0000", 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1221e-05 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652455+0000", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652498+0000", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3609000000000002e-05 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.652505+0000", 2024-08-17T17:46:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6309999999999997e-06 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601475+0000", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94897038600000005 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601485+0000", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2059999999999996e-06 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.812912+0000", 2024-08-17T17:46:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2114278860000001 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.813067+0000", 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015408300000000001 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.813257+0000", 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019044299999999999 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822946+0000", 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096887520000000001 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.822986+0000", 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0778000000000002e-05 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843268+0000", 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020281995000000001 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843308+0000", 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9626000000000003e-05 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849291+0000", 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059826929999999999 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124687 9.5 9:a52614fa:::benchmark_data_smithi064_275711_object124686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.93645798199998, 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1708211930000001, 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124687 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685128+0000", 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685129+0000", 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4600000000000004e-07 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685169+0000", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0126000000000002e-05 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685178+0000", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1200000000000008e-06 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685184+0000", 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000003e-06 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685356+0000", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171733 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.685366+0000", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0509999999999999e-05 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601513+0000", 2024-08-17T17:46:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91614678299999996 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601518+0000", 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9640000000000002e-06 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816519+0000", 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.215000675 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816637+0000", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118484 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.816738+0000", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101216 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826294+0000", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095560009999999997 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829584+0000", 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032897830000000001 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849468+0000", 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019883596 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849530+0000", 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1863e-05 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.855949+0000", 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064195329999999998 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124690 9.5 9:a653a86d:::benchmark_data_smithi064_275711_object124689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.930115903, 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1710445190000001, 2024-08-17T17:46:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124690 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691470+0000", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691472+0000", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691495+0000", 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.298e-05 2024-08-17T17:46:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691516+0000", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1622999999999999e-05 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691525+0000", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2130000000000008e-06 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691583+0000", 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8037e-05 2024-08-17T17:46:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.691592+0000", 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8570000000000007e-06 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601548+0000", 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90995501899999998 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601555+0000", 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6199999999999999e-06 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.819932+0000", 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218377241 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.820049+0000", 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116387 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.820150+0000", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101167 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829683+0000", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095330499999999995 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837074+0000", 2024-08-17T17:46:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073908680000000001 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856054+0000", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01898004 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856113+0000", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9218999999999997e-05 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862515+0000", 2024-08-17T17:46:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064015030000000002 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124707 9.5 9:a946d6e3:::benchmark_data_smithi064_275711_object124706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.91345415500001, 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.164556192, 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124707 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708132+0000", 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708133+0000", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.544e-06 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708156+0000", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665999999999999e-05 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708173+0000", 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6824999999999999e-05 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708181+0000", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9729999999999997e-06 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708243+0000", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2565999999999997e-05 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.708254+0000", 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0761000000000001e-05 2024-08-17T17:46:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601579+0000", 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89332489000000004 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601583+0000", 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4050000000000002e-06 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823020+0000", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2214364769999999 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823128+0000", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108113 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.823232+0000", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010371500000000001 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.837172+0000", 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013940111999999999 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843169+0000", 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059975000000000002 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.866982+0000", 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023812887000000001 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.867046+0000", 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4158000000000005e-05 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.872688+0000", 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056416000000000001 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124759 9.5 9:a81bd317:::benchmark_data_smithi064_275711_object124758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.85948616000002, 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.126767981, 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124759 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762100+0000", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762101+0000", 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.483e-06 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762130+0000", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8365e-05 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762146+0000", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6033999999999999e-05 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762154+0000", 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2470000000000008e-06 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762216+0000", 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2317000000000007e-05 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.762227+0000", 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063000000000001e-05 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601611+0000", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83938328900000003 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601618+0000", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4850000000000003e-06 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826349+0000", 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2247311299999999 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826489+0000", 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139801 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.826594+0000", 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105067 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843174+0000", 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016579611000000001 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843226+0000", 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2272999999999999e-05 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.880086+0000", 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036859801999999997 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.880158+0000", 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2336999999999995e-05 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888868+0000", 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087096770000000007 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124783 9.5 9:a6c522ba:::benchmark_data_smithi064_275711_object124782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.83265957100002, 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1068486069999999, 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124783 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788926+0000", 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788928+0000", 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.48e-06 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788950+0000", 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2152000000000001e-05 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788968+0000", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7768999999999999e-05 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.788976+0000", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0010000000000001e-06 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.789045+0000", 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9276000000000006e-05 2024-08-17T17:46:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.789055+0000", 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8300000000000008e-06 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601642+0000", 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81258676100000005 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601646+0000", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3440000000000003e-06 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829716+0000", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.228070405 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.829851+0000", 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000134363 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.830008+0000", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157267 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.843178+0000", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013170125 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849395+0000", 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062171759999999996 2024-08-17T17:46:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888981+0000", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039585596000000001 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.889041+0000", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0136000000000002e-05 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895775+0000", 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067339260000000003 2024-08-17T17:46:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124826 9.5 9:ae6b922f:::benchmark_data_smithi064_275711_object124825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.78524107700002, 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0662265, 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124826 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836345+0000", 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836347+0000", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9379999999999999e-06 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836375+0000", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8479999999999998e-05 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836397+0000", 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1815e-05 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836408+0000", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0366999999999999e-05 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836473+0000", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5469000000000006e-05 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.836485+0000", 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181e-05 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601666+0000", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76518151599999995 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601671+0000", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3830000000000002e-06 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.833771+0000", 2024-08-17T17:46:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.232100143 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.833943+0000", 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000172549 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.834055+0000", 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111354 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.849307+0000", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015252474 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.856008+0000", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067006920000000003 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895851+0000", 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039842620000000002 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.895919+0000", 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8134999999999997e-05 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902571+0000", 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066527549999999998 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124834 9.5 9:a84b73a1:::benchmark_data_smithi064_275711_object124833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.77566283599998, 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.063460681, 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124834 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845923+0000", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845925+0000", 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.719e-06 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845976+0000", 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0979000000000001e-05 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.845991+0000", 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4711e-05 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846000+0000", 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1179999999999998e-06 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846084+0000", 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4714000000000002e-05 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.846097+0000", 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2904999999999999e-05 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601692+0000", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75559463699999996 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601696+0000", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0029999999999996e-06 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862556+0000", 2024-08-17T17:46:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.260859795 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862739+0000", 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018281800000000001 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.862930+0000", 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000191917 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.888940+0000", 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026009211000000001 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902593+0000", 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013653439 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902638+0000", 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5309999999999998e-05 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.902687+0000", 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8667e-05 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909384+0000", 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066967379999999998 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124848 9.5 9:a16a5733:::benchmark_data_smithi064_275711_object124847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.76085429300002, 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1062764290000002, 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124848 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860732+0000", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860733+0000", 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.536e-06 2024-08-17T17:46:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860755+0000", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1763000000000002e-05 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860773+0000", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7655e-05 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860780+0000", 2024-08-17T17:46:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5429999999999996e-06 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860848+0000", 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7397999999999994e-05 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.860862+0000", 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.47e-05 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601726+0000", 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74086384599999999 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601735+0000", 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4339999999999992e-06 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909452+0000", 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3077175620000001 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909623+0000", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017124100000000001 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.909785+0000", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016205200000000001 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922612+0000", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012826561 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922629+0000", 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7314000000000001e-05 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.960337+0000", 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037707209999999998 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.960397+0000", 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0488000000000001e-05 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967008+0000", 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066111260000000002 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124851 9.5 9:a13810df:::benchmark_data_smithi064_275711_object124850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.75782462199999, 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.110583149, 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124851 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863761+0000", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863762+0000", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.09e-07 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863802+0000", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0132999999999997e-05 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863814+0000", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1408e-05 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863819+0000", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9189999999999998e-06 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863872+0000", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3310999999999998e-05 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.863878+0000", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3779999999999998e-06 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601806+0000", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73792795099999997 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601811+0000", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9230000000000001e-06 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914048+0000", 2024-08-17T17:46:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3122370210000001 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914205+0000", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000156334 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.914367+0000", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162497 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922635+0000", 2024-08-17T17:46:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082677570000000006 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929492+0000", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006857096 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967067+0000", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037574514000000003 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.967129+0000", 2024-08-17T17:46:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2793999999999995e-05 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974345+0000", 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072152040000000002 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124854 9.5 9:a175789f:::benchmark_data_smithi064_275711_object124853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:46:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.75347863399998, 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1131798100000001, 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124854 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:46:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868107+0000", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868109+0000", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156e-06 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868147+0000", 2024-08-17T17:46:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8748000000000002e-05 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868161+0000", 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3872e-05 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868169+0000", 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0129999999999993e-06 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868229+0000", 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0275000000000001e-05 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.868241+0000", 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1501999999999999e-05 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601831+0000", 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73359040200000003 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601836+0000", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9409999999999998e-06 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918294+0000", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.316458159 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918435+0000", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140469 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.918604+0000", 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168839 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929552+0000", 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010947811999999999 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929564+0000", 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2254e-05 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974488+0000", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044924015999999997 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.974556+0000", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8674e-05 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981287+0000", 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067306780000000004 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124857 9.5 9:aba14c59:::benchmark_data_smithi064_275711_object124856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.74839496599998, 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1145436690000001, 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124857 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873191+0000", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873193+0000", 2024-08-17T17:46:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.159e-06 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873235+0000", 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1999000000000002e-05 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873250+0000", 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4788e-05 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873257+0000", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8220000000000001e-06 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873311+0000", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4478000000000003e-05 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.873319+0000", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1540000000000002e-06 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601872+0000", 2024-08-17T17:46:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72855236599999995 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601876+0000", 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0940000000000001e-06 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922661+0000", 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3207853940000001 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922808+0000", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014638499999999999 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.922968+0000", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159889 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.929585+0000", 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066178030000000002 2024-08-17T17:46:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.931456+0000", 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001870771 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981424+0000", 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049967984 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.981488+0000", 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3699000000000001e-05 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987735+0000", 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062468840000000003 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:124863 9.5 9:a02aa46e:::benchmark_data_smithi064_275711_object124862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:46:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.72578245599999, 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0986319409999998, 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 124863 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895804+0000", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895805+0000", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.773e-06 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895826+0000", 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0367999999999999e-05 2024-08-17T17:46:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895840+0000", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4810999999999999e-05 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895847+0000", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7560000000000002e-06 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895913+0000", 2024-08-17T17:46:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5739000000000002e-05 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:30.895921+0000", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601901+0000", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70597918100000001 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:31.601906+0000", 2024-08-17T17:46:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5099999999999998e-06 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926428+0000", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3245222110000001 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926595+0000", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016628699999999999 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.926762+0000", 2024-08-17T17:46:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167252 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.933120+0000", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063576300000000004 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.944327+0000", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011207205 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987802+0000", 2024-08-17T17:46:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.043475371999999998 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.987925+0000", 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122485 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:32.994435+0000", 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065108700000000002 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:62789 10.4 10:2db4ee57:::benchmark_data_smithi064_306376_object62788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:45:24.127416+0000", 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.494170209000004, 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.50510335799999995, 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 62789 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127416+0000", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127416+0000", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127417+0000", 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2399999999999997e-07 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127437+0000", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0480999999999999e-05 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127450+0000", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2597e-05 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.127457+0000", 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0509999999999999e-06 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.130653+0000", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031958799999999999 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.130987+0000", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000334221 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.131151+0000", 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164232 2024-08-17T17:46:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.134555+0000", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003403943 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.137607+0000", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030516139999999998 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632396+0000", 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49478951300000001 2024-08-17T17:46:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632473+0000", 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6956e-05 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632519+0000", 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6045999999999999e-05 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:62794 10.4 10:257342bd:::benchmark_data_smithi064_306376_object62793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-08-17T17:46:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:45:24.133470+0000", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.488115925000002, 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49917281200000002, 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 62794 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133470+0000", 2024-08-17T17:46:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133470+0000", 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133472+0000", 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4389999999999999e-06 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133491+0000", 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9130000000000001e-05 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133507+0000", 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6595e-05 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.133514+0000", 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1010000000000003e-06 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.134725+0000", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012107419999999999 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.134862+0000", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013734399999999999 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.134993+0000", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013048500000000001 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.137620+0000", 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026268229999999999 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.139480+0000", 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001860118 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632565+0000", 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49308563300000002 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632605+0000", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9237999999999999e-05 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.632643+0000", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8164000000000003e-05 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:62805 10.4 10:2f4a1b3d:::benchmark_data_smithi064_306376_object62804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:45:24.142156+0000", 2024-08-17T17:46:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.479429647000003, 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49186354199999999, 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 62805 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:46:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142156+0000", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142156+0000", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142158+0000", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.544e-06 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:46:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142180+0000", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1727999999999999e-05 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142192+0000", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2255999999999999e-05 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142198+0000", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6790000000000002e-06 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:46:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142259+0000", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1478999999999999e-05 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142500+0000", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024105200000000001 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.142631+0000", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131144 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:46:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.145375+0000", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002743812 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.145692+0000", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031672500000000001 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.633903+0000", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48821087099999999 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:46:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.633981+0000", 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8416000000000003e-05 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.634020+0000", 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8835999999999999e-05 2024-08-17T17:46:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:46:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:46:45.526 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:46:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:45.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:45.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:45.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:45.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:45.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:45.910 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:45.948 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:46.050 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:46.095 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:46.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:46.237 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:46.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:46.372 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:46.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:46.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:46.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.643 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:46.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:46.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:46.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:46.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:46.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:46 smithi175 ceph-mon[101434]: pgmap v1408: 99 pgs: 99 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:46:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.922 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:46.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:46.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:46 smithi064 ceph-mon[104516]: pgmap v1408: 99 pgs: 99 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:46:46.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:46 smithi064 ceph-mon[109246]: pgmap v1408: 99 pgs: 99 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:46:47.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:47.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:47.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:47.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:47.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:47.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:47.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:47.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:47.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:46:47.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:47.522 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:47.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:47.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:47.652 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:47.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:47.792 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:47.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:48.030 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:48.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:48.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:48.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:48.201 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:48.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:48.347 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:48.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:48.457 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.509 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:48.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:48.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.672 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:48.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:48.773 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:48.806 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:48.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:48 smithi175 ceph-mon[101434]: pgmap v1409: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:46:48.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:48 smithi064 ceph-mon[104516]: pgmap v1409: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:46:48.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:48 smithi064 ceph-mon[109246]: pgmap v1409: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:46:48.947 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:49.049 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:49.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:49.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:49.224 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:49.259 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:49.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:49.357 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:49.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:49.459 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:49.474 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:46:49.479 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pg_num to 26 2024-08-17T17:46:49.500 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:49.602 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:49.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:49.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:49.768 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:49.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:49.915 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3592972194' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[104516]: osdmap e436: 8 total, 8 up, 7 in 2024-08-17T17:46:49.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3592972194' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:49.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:49 smithi064 ceph-mon[109246]: osdmap e436: 8 total, 8 up, 7 in 2024-08-17T17:46:50.018 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:50.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:50.157 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:50.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3592972194' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2781433923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:50.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:49 smithi175 ceph-mon[101434]: osdmap e436: 8 total, 8 up, 7 in 2024-08-17T17:46:50.195 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:50.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:50.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:50.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:50.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:50.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:50.594 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:50.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:50.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:50.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:50.842 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:50.877 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:50 smithi064 ceph-mon[104516]: pgmap v1410: 99 pgs: 99 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:46:50.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:50 smithi064 ceph-mon[109246]: pgmap v1410: 99 pgs: 99 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:46:50.979 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:51.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:51.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:51.155 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:51.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:50 smithi175 ceph-mon[101434]: pgmap v1410: 99 pgs: 99 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 940 op/s 2024-08-17T17:46:51.257 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:51.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:51.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:51.434 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:51.536 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:51.574 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:51.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:51.728 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:51.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:51.886 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:51 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:46:51.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:51 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:46:51.988 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.035 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:52.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:52.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:52.157 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:52.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:51 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:46:52.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:52.280 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:52.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:52.405 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:52.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:52.530 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:52.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:52.654 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:52.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:52.780 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:52.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:46:52.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[104516]: pgmap v1412: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:46:52.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:46:52.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[104516]: osdmap e437: 8 total, 8 up, 7 in 2024-08-17T17:46:52.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[109246]: pgmap v1412: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:46:52.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:46:52.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:52 smithi064 ceph-mon[109246]: osdmap e437: 8 total, 8 up, 7 in 2024-08-17T17:46:52.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:52.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:53.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:53.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:53.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:53.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:52 smithi175 ceph-mon[101434]: pgmap v1412: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:46:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:52 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:46:53.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:52 smithi175 ceph-mon[101434]: osdmap e437: 8 total, 8 up, 7 in 2024-08-17T17:46:53.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:53.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:53.309 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:53.343 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:53.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:53.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:53.478 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:53.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:53.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:53.713 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:53.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:53.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:53.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:53.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:54.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:54.117 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:54.152 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:54.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:53 smithi175 ceph-mon[101434]: osdmap e438: 8 total, 8 up, 7 in 2024-08-17T17:46:54.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:53 smithi064 ceph-mon[104516]: osdmap e438: 8 total, 8 up, 7 in 2024-08-17T17:46:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:46:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:46:53] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-08-17T17:46:54.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:53 smithi064 ceph-mon[109246]: osdmap e438: 8 total, 8 up, 7 in 2024-08-17T17:46:54.253 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:54.291 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:54.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:54.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:54.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:54.560 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:54.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:54.695 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:54.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.834 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:54.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:54.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:54.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[104516]: pgmap v1415: 109 pgs: 10 unknown, 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 858 op/s 2024-08-17T17:46:54.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:46:54.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[104516]: 10.2 scrub starts 2024-08-17T17:46:54.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[109246]: pgmap v1415: 109 pgs: 10 unknown, 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 858 op/s 2024-08-17T17:46:54.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:46:54.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:54 smithi064 ceph-mon[109246]: 10.2 scrub starts 2024-08-17T17:46:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.968 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:54.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:55.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:55.104 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:54 smithi175 ceph-mon[101434]: pgmap v1415: 109 pgs: 10 unknown, 99 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 858 op/s 2024-08-17T17:46:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:54 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:46:55.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:54 smithi175 ceph-mon[101434]: 10.2 scrub starts 2024-08-17T17:46:55.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:55.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:55.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:55.559 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:55.661 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:55.699 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:55.702 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:55.703 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:55.754 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:55.755 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:55.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:46:55.773 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:55.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:55.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:46:55.810 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:55.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:55.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[104516]: 10.8 scrub starts 2024-08-17T17:46:55.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[104516]: 10.12 scrub starts 2024-08-17T17:46:55.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/4191406099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[109246]: 10.8 scrub starts 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[109246]: 10.12 scrub starts 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:55.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/4191406099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:55.844 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:46:55.845 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:55.941 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:55.984 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:56.086 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.123 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:56.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:55 smithi175 ceph-mon[101434]: 10.8 scrub starts 2024-08-17T17:46:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:55 smithi175 ceph-mon[101434]: 10.12 scrub starts 2024-08-17T17:46:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:46:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:56.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/4191406099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:56.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:56.255 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:56.357 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:56.398 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:56.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:46:56.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:56.574 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:56.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:56.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:56.810 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:56.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:56.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:56.990 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:56.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:56.993 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:56 smithi064 ceph-mon[104516]: pgmap v1416: 109 pgs: 9 unknown, 100 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:56.993 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:56 smithi064 ceph-mon[104516]: 10.15 scrub starts 2024-08-17T17:46:56.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:56 smithi064 ceph-mon[109246]: pgmap v1416: 109 pgs: 9 unknown, 100 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:56.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:56 smithi064 ceph-mon[109246]: 10.15 scrub starts 2024-08-17T17:46:57.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:57.128 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:57.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:57.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:56 smithi175 ceph-mon[101434]: pgmap v1416: 109 pgs: 9 unknown, 100 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:57.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:56 smithi175 ceph-mon[101434]: 10.15 scrub starts 2024-08-17T17:46:57.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.266 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:57.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:57.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.398 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:57.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.638 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:57.678 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:57.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:57.779 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:57.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:57.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:58.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:57 smithi064 ceph-mon[104516]: 10.4 scrub starts 2024-08-17T17:46:58.020 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:57 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:46:58.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:57 smithi064 ceph-mon[109246]: 10.4 scrub starts 2024-08-17T17:46:58.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:57 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:46:58.052 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:58.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:58.086 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:58.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:57 smithi175 ceph-mon[101434]: 10.4 scrub starts 2024-08-17T17:46:58.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:57 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:46:58.189 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:58.210 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:58.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:58.334 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:58.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:58.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:58.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:58.583 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:58.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:58.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:58.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:58.809 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:58.835 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:58.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:58.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:59.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:58 smithi175 ceph-mon[101434]: pgmap v1417: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:58 smithi175 ceph-mon[101434]: 10.18 scrub starts 2024-08-17T17:46:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:58 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:46:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:58 smithi175 ceph-mon[101434]: osdmap e439: 8 total, 8 up, 7 in 2024-08-17T17:46:59.062 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:58 smithi175 ceph-mon[101434]: osdmap e440: 8 total, 8 up, 7 in 2024-08-17T17:46:59.084 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:59.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[104516]: pgmap v1417: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:59.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[104516]: 10.18 scrub starts 2024-08-17T17:46:59.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:46:59.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[104516]: osdmap e439: 8 total, 8 up, 7 in 2024-08-17T17:46:59.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[104516]: osdmap e440: 8 total, 8 up, 7 in 2024-08-17T17:46:59.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[109246]: pgmap v1417: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:46:59.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[109246]: 10.18 scrub starts 2024-08-17T17:46:59.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:46:59.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[109246]: osdmap e439: 8 total, 8 up, 7 in 2024-08-17T17:46:59.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:58 smithi064 ceph-mon[109246]: osdmap e440: 8 total, 8 up, 7 in 2024-08-17T17:46:59.186 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:59.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:46:59.211 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:59.313 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:59.338 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:59.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:59.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:46:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:59.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:59.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:59.708 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:59.743 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:59.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:59.817 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[104516]: 10.18 scrub starts 2024-08-17T17:46:59.817 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[104516]: 10.2 scrub starts 2024-08-17T17:46:59.817 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:46:59.817 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:46:59.817 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[104516]: osdmap e441: 8 total, 8 up, 7 in 2024-08-17T17:46:59.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[109246]: 10.18 scrub starts 2024-08-17T17:46:59.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[109246]: 10.2 scrub starts 2024-08-17T17:46:59.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:46:59.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:46:59.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:46:59 smithi064 ceph-mon[109246]: osdmap e441: 8 total, 8 up, 7 in 2024-08-17T17:46:59.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:59.875 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:46:59.875 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:46:59.875 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-08-17T17:46:59.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd in 3 2024-08-17T17:46:59.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:46:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:59.981 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:00.011 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:00.113 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:00.144 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:00.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:59 smithi175 ceph-mon[101434]: 10.18 scrub starts 2024-08-17T17:47:00.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:59 smithi175 ceph-mon[101434]: 10.2 scrub starts 2024-08-17T17:47:00.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:47:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:47:00.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:46:59 smithi175 ceph-mon[101434]: osdmap e441: 8 total, 8 up, 7 in 2024-08-17T17:47:00.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:00.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:00.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:00.370 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:00.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:00.474 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:47:00.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:00.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:00.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:00.642 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:00.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:00.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:00.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:00.889 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:00.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:01.013 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:01.108 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: pgmap v1420: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: 10.8 scrub starts 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9774 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: 10.12 scrub starts 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[104516]: osdmap e442: 8 total, 8 up, 7 in 2024-08-17T17:47:01.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: pgmap v1420: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: 10.8 scrub starts 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9774 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: 10.12 scrub starts 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:01.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:00 smithi064 ceph-mon[109246]: osdmap e442: 8 total, 8 up, 7 in 2024-08-17T17:47:01.115 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:01.139 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:01.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: pgmap v1420: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:47:01.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: 10.8 scrub starts 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 438'9774 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: 10.12 scrub starts 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1165636465' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:01.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:00 smithi175 ceph-mon[101434]: osdmap e442: 8 total, 8 up, 7 in 2024-08-17T17:47:01.241 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.264 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:01.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:01.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:01.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:01.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:01.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:01.653 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:01.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:01.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:01.889 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:01.920 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:02.022 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:02.048 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:02.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:47:02.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:47:02.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[104516]: osdmap e443: 8 total, 8 up, 7 in 2024-08-17T17:47:02.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:47:02.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:47:02.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:01 smithi064 ceph-mon[109246]: osdmap e443: 8 total, 8 up, 7 in 2024-08-17T17:47:02.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:47:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:47:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:01 smithi175 ceph-mon[101434]: osdmap e443: 8 total, 8 up, 7 in 2024-08-17T17:47:02.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:02.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:02.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:02.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:02.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:02.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:02.553 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:02.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:02.678 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:02.704 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:02.704 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:02.779 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:02.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[104516]: pgmap v1423: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 2103/272451 objects misplaced (0.772%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:02.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[104516]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 439'9638 2024-08-17T17:47:02.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[104516]: osdmap e444: 8 total, 8 up, 7 in 2024-08-17T17:47:02.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[109246]: pgmap v1423: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 2103/272451 objects misplaced (0.772%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:02.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[109246]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 439'9638 2024-08-17T17:47:02.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:02 smithi064 ceph-mon[109246]: osdmap e444: 8 total, 8 up, 7 in 2024-08-17T17:47:02.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:02.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:02.928 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:02.945 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:03.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:03.055 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:03.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:02 smithi175 ceph-mon[101434]: pgmap v1423: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 746 op/s; 2103/272451 objects misplaced (0.772%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:02 smithi175 ceph-mon[101434]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 439'9638 2024-08-17T17:47:03.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:02 smithi175 ceph-mon[101434]: osdmap e444: 8 total, 8 up, 7 in 2024-08-17T17:47:03.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:03.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:03.310 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:03.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:03.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:03.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:03.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:03.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:03.686 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:03.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:03.812 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:03.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:47:03.914 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:03.937 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:04.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:04.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:04.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:04.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:03] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.51.0" 2024-08-17T17:47:04.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:04.209 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:04.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:04.310 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:04.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:04.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.491 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:04.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:04.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:47:04.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:04.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:04.724 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:04.761 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:04.862 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:04.907 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:04 smithi175 ceph-mon[101434]: pgmap v1426: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 4902/269754 objects misplaced (1.817%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:04.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:04 smithi175 ceph-mon[101434]: osdmap e445: 8 total, 8 up, 7 in 2024-08-17T17:47:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:04 smithi175 ceph-mon[101434]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:04 smithi175 ceph-mon[101434]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:04 smithi175 ceph-mon[101434]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[104516]: pgmap v1426: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 4902/269754 objects misplaced (1.817%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[104516]: osdmap e445: 8 total, 8 up, 7 in 2024-08-17T17:47:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[104516]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[104516]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[104516]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[109246]: pgmap v1426: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 4902/269754 objects misplaced (1.817%); 11 MiB/s, 181 objects/s recovering 2024-08-17T17:47:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[109246]: osdmap e445: 8 total, 8 up, 7 in 2024-08-17T17:47:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[109246]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[109246]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:04 smithi064 ceph-mon[109246]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 441'9726 2024-08-17T17:47:05.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.048 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:05.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:05.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:05.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:05.310 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:05.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.450 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:05.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:05.588 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:05.660 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:05.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:05.737 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:05.839 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:05.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:05.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:06.011 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:06.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.145 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:06.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:06.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:06.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:06.384 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:06.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:06.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:06.561 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:06.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:06.663 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:06.700 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:06.801 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:06.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:06 smithi064 ceph-mon[104516]: pgmap v1428: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 4902/268983 objects misplaced (1.822%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:47:06.805 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:06 smithi064 ceph-mon[109246]: pgmap v1428: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 4902/268983 objects misplaced (1.822%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:47:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.840 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:06.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:06 smithi175 ceph-mon[101434]: pgmap v1428: 109 pgs: 1 activating+remapped, 1 active+remapped+backfilling, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 4902/268983 objects misplaced (1.822%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:47:06.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:06.975 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:07.076 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:07.079 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:06 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:47:06.802+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e445 definitely_dead 0 2024-08-17T17:47:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.113 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:07.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:07.214 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:07.253 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:07.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:07.386 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:07.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:07.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:07.514 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.3. 2024-08-17T17:47:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.540 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:07.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:07.641 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:07.663 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:07.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3148426123' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:47:07.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:07.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3148426123' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:47:07.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:07 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:07.765 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:07.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:47:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:47:07.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:07.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:07.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:07.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:07 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3148426123' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:07 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-08-17T17:47:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:47:07.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:07 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:07.913 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:08.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:08.037 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:08.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:08.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:08.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:08.285 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:08.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.410 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.536 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:08.661 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:08.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: pgmap v1429: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 9441/260016 objects misplaced (3.631%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:08.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2114579999' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[109246]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: pgmap v1429: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 9441/260016 objects misplaced (3.631%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2114579999' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:08.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:08 smithi064 ceph-mon[104516]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:47:08.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:08.909 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:08.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: pgmap v1429: 109 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 106 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 9441/260016 objects misplaced (3.631%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-08-17T17:47:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:47:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:47:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2114579999' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:08.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:08 smithi175 ceph-mon[101434]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:47:08.947 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-08-17T17:47:09.011 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:09.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:09.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:09.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:09.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:09.286 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:09.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:09.412 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:09.514 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:09.538 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:09.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:09.667 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:09 smithi064 ceph-mon[104516]: 10.18 scrub starts 2024-08-17T17:47:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:09 smithi064 ceph-mon[104516]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:47:09.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:09 smithi064 ceph-mon[109246]: 10.18 scrub starts 2024-08-17T17:47:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:09 smithi064 ceph-mon[109246]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:47:09.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:09.811 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:09.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:09.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:09 smithi175 ceph-mon[101434]: 10.18 scrub starts 2024-08-17T17:47:09.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:09 smithi175 ceph-mon[101434]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:47:09.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:09.944 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:09.991 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:10.046 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:10.079 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:10.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.226 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:10.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:10.327 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:10.355 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:10.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:10.498 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:10.518 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:47:10.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:10.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:10.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:10.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:10.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:10.888 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: pgmap v1432: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 9441/257985 objects misplaced (3.660%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.8 scrub starts 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.17 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9829 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9846 2024-08-17T17:47:10.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9889 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.14 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9941 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9873 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.15 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9946 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 445'10385 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9962 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/284265644' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:10.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:10.923 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: pgmap v1432: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 9441/257985 objects misplaced (3.660%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.8 scrub starts 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.17 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9829 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9846 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9889 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.14 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9941 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9873 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.15 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9946 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 445'10385 2024-08-17T17:47:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9962 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/284265644' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: pgmap v1432: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 105 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 9441/257985 objects misplaced (3.660%); 36 MiB/s, 581 objects/s recovering 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.8 scrub starts 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.17 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9829 2024-08-17T17:47:10.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9846 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9889 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.14 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9941 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9873 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 445'9882 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.15 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9946 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 445'10385 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 445'9962 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/284265644' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:10.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:11.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:11.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:11.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:11.161 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:11.216 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:11.318 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:11.366 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:11.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:11.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:11.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:11.636 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[104516]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:11.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:11 smithi064 ceph-mon[109246]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:47:11.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:11.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:47:11.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.776 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:11.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:11.857 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:11 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:11.857 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:11 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:11.857 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:11 smithi175 ceph-mon[101434]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:47:11.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:11.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:12.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.067 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:12.167 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:12.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:12.320 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:12.357 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:12.460 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.517 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:12.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:12.651 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:12.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:12.783 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:12.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:12 smithi064 ceph-mon[104516]: pgmap v1435: 109 pgs: 2 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 104 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 10468/254247 objects misplaced (4.117%); 23 MiB/s, 377 objects/s recovering 2024-08-17T17:47:12.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:12 smithi064 ceph-mon[104516]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:47:12.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:12 smithi064 ceph-mon[109246]: pgmap v1435: 109 pgs: 2 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 104 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 10468/254247 objects misplaced (4.117%); 23 MiB/s, 377 objects/s recovering 2024-08-17T17:47:12.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:12 smithi064 ceph-mon[109246]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169987 8.10 8:089c48ca:::benchmark_data_smithi064_242541_object47494:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:47:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92655218599998, 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2428539119999999, 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169987 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903870+0000", 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9499999999999998e-07 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903872+0000", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903877+0000", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9439999999999996e-06 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903881+0000", 2024-08-17T17:47:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0589999999999996e-06 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904532+0000", 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065159099999999998 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904539+0000", 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2670000000000003e-06 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.108495+0000", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2039566210000001 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.108618+0000", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012227700000000001 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.109107+0000", 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048971300000000002 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117836+0000", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087290200000000005 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124586+0000", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067496830000000002 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130499+0000", 2024-08-17T17:47:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059131540000000003 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130541+0000", 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2181999999999997e-05 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146530+0000", 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015988176999999999 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146592+0000", 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2388e-05 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146724+0000", 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131812 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169991 8.10 8:0d6229e0:::benchmark_data_smithi064_242541_object47498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92652514199995, 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2432334940000001, 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169991 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903897+0000", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9999999999999998e-07 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903899+0000", 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9319999999999999e-06 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903905+0000", 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8629999999999996e-06 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903910+0000", 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7380000000000002e-06 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904561+0000", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065160999999999995 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904568+0000", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5899999999999996e-06 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.114585+0000", 2024-08-17T17:47:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2100170699999999 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.114710+0000", 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012532 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.115118+0000", 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040799299999999999 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117895+0000", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027765450000000001 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124599+0000", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067038530000000001 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130588+0000", 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059897450000000003 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130681+0000", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2915000000000001e-05 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146948+0000", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016266138999999999 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147007+0000", 2024-08-17T17:47:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9379999999999997e-05 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147130+0000", 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012340100000000001 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:169992 8.10 8:0e67e8fd:::benchmark_data_smithi064_242541_object47499:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92650757000001, 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.243508251, 2024-08-17T17:47:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169992 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903914+0000", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903915+0000", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2300000000000002e-07 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903916+0000", 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.345e-06 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903937+0000", 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1189999999999999e-05 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903942+0000", 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6800000000000001e-06 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904593+0000", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065059899999999997 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904599+0000", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5289999999999997e-06 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.117967+0000", 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.213367941 2024-08-17T17:47:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.118102+0000", 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135067 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.118638+0000", 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053601699999999996 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124526+0000", 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058880119999999998 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124606+0000", 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0197000000000005e-05 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130771+0000", 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061646030000000003 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146771+0000", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016000316000000001 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147224+0000", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000452686 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147281+0000", 2024-08-17T17:47:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7136999999999999e-05 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147423+0000", 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014160900000000001 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170001 8.10 8:0f85a16f:::benchmark_data_smithi064_242541_object47508:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92647492399999, 2024-08-17T17:47:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244356507, 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170001 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903947+0000", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2899999999999999e-07 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903949+0000", 2024-08-17T17:47:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.635e-06 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903954+0000", 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4399999999999996e-06 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903959+0000", 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1250000000000003e-06 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904625+0000", 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066614300000000001 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904631+0000", 2024-08-17T17:47:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.02e-06 2024-08-17T17:47:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.121580+0000", 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.216949662 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.121845+0000", 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026459999999999998 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.122273+0000", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042810299999999999 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124574+0000", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002301179 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130816+0000", 2024-08-17T17:47:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062412489999999999 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147177+0000", 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016361550999999998 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147515+0000", 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000338174 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148054+0000", 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053849400000000004 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148108+0000", 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4320000000000002e-05 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148304+0000", 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019548300000000001 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170081 8.10 8:0e59af9f:::benchmark_data_smithi064_242541_object47588:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92645894500004, 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2447100250000001, 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170081 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903963+0000", 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2099999999999998e-07 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903965+0000", 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4219999999999999e-06 2024-08-17T17:47:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903971+0000", 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6619999999999999e-06 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903976+0000", 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.155e-06 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904655+0000", 2024-08-17T17:47:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067936500000000003 2024-08-17T17:47:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904662+0000", 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.294e-06 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124647+0000", 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.219984261 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.124737+0000", 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0848999999999997e-05 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.125056+0000", 2024-08-17T17:47:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031911 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130634+0000", 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055775920000000001 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146485+0000", 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015850578000000001 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147467+0000", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098212400000000002 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147602+0000", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135482 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148496+0000", 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00089352699999999997 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148549+0000", 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2936000000000003e-05 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148673+0000", 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012434699999999999 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170094 8.10 8:08c04a44:::benchmark_data_smithi064_242541_object47601:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92644191500005, 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244854007, 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170094 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903980+0000", 2024-08-17T17:47:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9200000000000002e-07 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903982+0000", 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8029999999999999e-06 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903987+0000", 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1660000000000002e-06 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903991+0000", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1699999999999999e-06 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904688+0000", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069637300000000002 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904695+0000", 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7429999999999996e-06 2024-08-17T17:47:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.126968+0000", 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2222730880000001 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.127086+0000", 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118687 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.127420+0000", 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033324199999999998 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.130726+0000", 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033066710000000002 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146901+0000", 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016175168 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147562+0000", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066013900000000002 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.147642+0000", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0129999999999993e-05 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148680+0000", 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001038073 2024-08-17T17:47:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148724+0000", 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4008999999999998e-05 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148834+0000", 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110253 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170129 8.10 8:08f27c42:::benchmark_data_smithi064_242541_object47636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92642607000005, 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2624799170000001, 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170129 2024-08-17T17:47:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903996+0000", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9799999999999999e-07 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.903998+0000", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6300000000000001e-06 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904004+0000", 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4670000000000001e-06 2024-08-17T17:47:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904009+0000", 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3660000000000002e-06 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904719+0000", 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000710591 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904726+0000", 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5130000000000002e-06 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143356+0000", 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.238629918 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143482+0000", 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012597899999999999 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.143836+0000", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035482299999999999 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.146841+0000", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003004456 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148040+0000", 2024-08-17T17:47:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011986900000000001 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.148840+0000", 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080081300000000005 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166184+0000", 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017343715999999999 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166291+0000", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106687 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166351+0000", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0102000000000002e-05 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166476+0000", 2024-08-17T17:47:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012486800000000001 2024-08-17T17:47:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.895 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170285 8.10 8:0860b59f:::benchmark_data_smithi064_242541_object47792:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92640898599996, 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2635653019999999, 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170285 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904013+0000", 2024-08-17T17:47:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2500000000000001e-07 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904019+0000", 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3920000000000002e-06 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904026+0000", 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9260000000000003e-06 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904030+0000", 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3340000000000003e-06 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904752+0000", 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072161000000000002 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904758+0000", 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6869999999999997e-06 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.151809+0000", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.247050346 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.151945+0000", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013669299999999999 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.152345+0000", 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000399527 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163133+0000", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010787711 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.165979+0000", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028468270000000001 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166243+0000", 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026321800000000001 2024-08-17T17:47:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166559+0000", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031634700000000003 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166641+0000", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2047999999999998e-05 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167435+0000", 2024-08-17T17:47:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079408199999999999 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167578+0000", 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014322900000000001 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170286 8.10 8:0ea5b216:::benchmark_data_smithi064_242541_object47793:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:47:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92638726500002, 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2641885799999999, 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170286 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:47:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904035+0000", 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1300000000000001e-07 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904037+0000", 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.689e-06 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904043+0000", 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4529999999999999e-06 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904048+0000", 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7090000000000001e-06 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904783+0000", 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073487800000000001 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904789+0000", 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2400000000000004e-06 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.154791+0000", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.250002142 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.154906+0000", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114443 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.155276+0000", 2024-08-17T17:47:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037086200000000001 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.165936+0000", 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010659221999999999 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166518+0000", 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058253700000000005 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.166600+0000", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1546999999999997e-05 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167623+0000", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010228850000000001 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167861+0000", 2024-08-17T17:47:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000238784 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.167925+0000", 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4028000000000004e-05 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.168223+0000", 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029784800000000003 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170365 8.10 8:08eecb7e:::benchmark_data_smithi064_242541_object47872:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92636958000003, 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.276395232, 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170365 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904052+0000", 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904053+0000", 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7399999999999999e-07 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904054+0000", 2024-08-17T17:47:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.511e-06 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904059+0000", 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1429999999999997e-06 2024-08-17T17:47:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904069+0000", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6190000000000006e-06 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904812+0000", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00074339899999999995 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904819+0000", 2024-08-17T17:47:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1909999999999997e-06 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163165+0000", 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2583468250000001 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163287+0000", 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012170100000000001 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.163679+0000", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039162499999999999 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.168831+0000", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051521199999999996 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172125+0000", 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032937069999999999 2024-08-17T17:47:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172269+0000", 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014436700000000001 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.176836+0000", 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045666930000000001 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180269+0000", 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034337500000000002 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180323+0000", 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3349e-05 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.180448+0000", 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495799999999999 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:170367 8.10 8:0f498d28:::benchmark_data_smithi064_242541_object47874:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 579.92634847500005, 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2886117530000001, 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:47:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 170367 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904074+0000", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.03e-07 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904075+0000", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.398e-06 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904085+0000", 2024-08-17T17:47:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4590000000000004e-06 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904089+0000", 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904905+0000", 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081554800000000003 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:32.904912+0000", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1600000000000001e-06 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172352+0000", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.267439948 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.172559+0000", 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020787099999999999 2024-08-17T17:47:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5,7", 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.173288+0000", 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072807099999999995 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.181788+0000", 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085003179999999998 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.182960+0000", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011723339999999999 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.183025+0000", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4449999999999994e-05 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.184212+0000", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011870310000000001 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192457+0000", 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082452319999999999 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192537+0000", 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0438000000000005e-05 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:37:34.192685+0000", 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000147892 2024-08-17T17:47:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143320 9.6 e397/375 9:66dc7620:::benchmark_data_smithi064_275711_object18325:head v 397'8971, mlcod=397'8968)", 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.97337667900001, 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36413622600000001, 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857045+0000", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857046+0000", 2024-08-17T17:47:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8099999999999999e-07 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857049+0000", 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.579e-06 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857055+0000", 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.122e-06 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.857060+0000", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3170000000000003e-06 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.859788+0000", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027285529999999999 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.859804+0000", 2024-08-17T17:47:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6273000000000002e-05 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221126+0000", 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36132131200000001 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221182+0000", 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5788999999999997e-05 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143328 9.6 e397/375 9:68bcc8f8:::benchmark_data_smithi064_275711_object18333:head v 397'8972, mlcod=397'8968)", 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.96710508799998, 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35789923299999998, 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:47:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863317+0000", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863318+0000", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.564e-06 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863326+0000", 2024-08-17T17:47:12.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9959999999999993e-06 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863341+0000", 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4409999999999999e-05 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.863349+0000", 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0430000000000008e-06 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.872140+0000", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087913839999999993 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.872219+0000", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8678999999999997e-05 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221190+0000", 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34897107399999999 2024-08-17T17:47:12.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221216+0000", 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6083000000000001e-05 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:143330 9.6 e397/375 9:6a00a6c4:::benchmark_data_smithi064_275711_object18335:head v 397'8973, mlcod=397'8968)", 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.96206457300002, 2024-08-17T17:47:12.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35288467099999998, 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868357+0000", 2024-08-17T17:47:12.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.943 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:12 smithi175 ceph-mon[101434]: pgmap v1435: 109 pgs: 2 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 104 active+clean; 5.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 10468/254247 objects misplaced (4.117%); 23 MiB/s, 377 objects/s recovering 2024-08-17T17:47:12.943 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:12 smithi175 ceph-mon[101434]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868358+0000", 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0100000000000005e-07 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868364+0000", 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7679999999999997e-06 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868374+0000", 2024-08-17T17:47:12.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0087e-05 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.868381+0000", 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7689999999999999e-06 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881427+0000", 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013046713 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881449+0000", 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1970000000000001e-05 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221219+0000", 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33976983399999999 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221242+0000", 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2929e-05 2024-08-17T17:47:12.946 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143338 9.0 9:0157aaa0:::benchmark_data_smithi064_275711_object18343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.95417302300001, 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34528550899999999, 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.947 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143338 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876249+0000", 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876251+0000", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9989999999999998e-06 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876256+0000", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876270+0000", 2024-08-17T17:47:12.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4054e-05 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876278+0000", 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3469999999999999e-06 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876879+0000", 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060110999999999997 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876989+0000", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110305 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.877163+0000", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017362599999999999 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884715+0000", 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075518879999999997 2024-08-17T17:47:12.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884860+0000", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014478099999999999 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221121+0000", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.336261635 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221235+0000", 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011311600000000001 2024-08-17T17:47:12.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221534+0000", 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029992899999999999 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143341 9.0 9:0b34feb9:::benchmark_data_smithi064_275711_object18346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.95352224800001, 2024-08-17T17:47:12.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34476353100000001, 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143341 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:47:12.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876900+0000", 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876901+0000", 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.035e-06 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876906+0000", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9690000000000002e-06 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876917+0000", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1211e-05 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.876924+0000", 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7000000000000002e-06 2024-08-17T17:47:12.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880772+0000", 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038480379999999998 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.880868+0000", 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6372000000000005e-05 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:47:12.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.881067+0000", 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019910500000000001 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.884867+0000", 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003799714 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.885267+0000", 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039973899999999998 2024-08-17T17:47:12.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221584+0000", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.336317596 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221630+0000", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5741999999999997e-05 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.221663+0000", 2024-08-17T17:47:12.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3309999999999998e-05 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45827.0:143347 9.0 9:002cf2b1:::benchmark_data_smithi064_275711_object18352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 316.93990854700002, 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.349745097, 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.960 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45827", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/527062778", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143347 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890513+0000", 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890515+0000", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3790000000000001e-06 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:55.890519+0000", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4490000000000001e-06 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.225664+0000", 2024-08-17T17:47:12.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.335145111 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.225681+0000", 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7027000000000001e-05 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226202+0000", 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052065800000000001 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226304+0000", 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101618 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.226412+0000", 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010857900000000001 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.238847+0000", 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012434822999999999 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.238864+0000", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6684000000000002e-05 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240165+0000", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013013639999999999 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240218+0000", 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2899999999999998e-05 2024-08-17T17:47:12.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:41:56.240259+0000", 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0504999999999999e-05 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:62811 10.2 10:41e0949a:::benchmark_data_smithi064_306376_object62810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:45:24.151537+0000", 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 108.67888496899999, 2024-08-17T17:47:12.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47865269799999999, 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 62811 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151537+0000", 2024-08-17T17:47:12.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151537+0000", 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151539+0000", 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.86e-06 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151561+0000", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.177e-05 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151576+0000", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5726000000000002e-05 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151583+0000", 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.002e-06 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151651+0000", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7181000000000006e-05 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151826+0000", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017505400000000001 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.151995+0000", 2024-08-17T17:47:12.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000169514 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.153908+0000", 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019132769999999999 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.154347+0000", 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043904799999999999 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.630111+0000", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47576373500000002 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.630155+0000", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3875999999999998e-05 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:45:24.630190+0000", 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4654999999999999e-05 2024-08-17T17:47:12.972 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:151807 10.2 10:41f7a2db:::benchmark_data_smithi064_306376_object26826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:46:51.701980+0000", 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 21.128441493, 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.029764379, 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:47:12.973 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151807 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.701980+0000", 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.701980+0000", 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.701983+0000", 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7989999999999998e-06 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.702008+0000", 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4624999999999999e-05 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.702037+0000", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.881e-05 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.702050+0000", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3283e-05 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.702129+0000", 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8710999999999999e-05 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.702135+0000", 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0299999999999999e-06 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.852318+0000", 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15018298899999999 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.852328+0000", 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0212e-05 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.731691+0000", 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87936299399999995 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.731745+0000", 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3925999999999998e-05 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.978 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:151811 10.2 10:45b51ca2:::benchmark_data_smithi064_306376_object26830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:46:51.718186+0000", 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 21.112236258999999, 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028470446, 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 151811 2024-08-17T17:47:12.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718186+0000", 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718186+0000", 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:12.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718189+0000", 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5719999999999999e-06 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718202+0000", 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2335e-05 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718233+0000", 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1832999999999998e-05 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.718247+0000", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3376e-05 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.852350+0000", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.134102903 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:51.852357+0000", 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7970000000000003e-06 2024-08-17T17:47:12.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.732182+0000", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87982525099999997 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.732500+0000", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031776800000000001 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.732917+0000", 2024-08-17T17:47:12.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041781899999999999 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.734682+0000", 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017650070000000001 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.738187+0000", 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003505022 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.746476+0000", 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082883599999999998 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.746555+0000", 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9380000000000002e-05 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:52.746656+0000", 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101023 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.985 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:12.986 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:12.986 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:47:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:12.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:13.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:13.144 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:13.246 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.281 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:13.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:13.382 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:13.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:13.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:13.559 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:13.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:13.706 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:13.808 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:13.842 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:13.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:13.948 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:47:13.948 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:13.948 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-17T17:47:13.948 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:47:13.948 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:47:13.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-17T17:47:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:13.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:14.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:47:14.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.158 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:14.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:14.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:14.298 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:47:14.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:14.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:14.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:13] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-08-17T17:47:14.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:14.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:14.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:14.581 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:14.682 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:14.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:14.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:14 smithi175 ceph-mon[101434]: pgmap v1437: 109 pgs: 1 activating, 2 peering, 1 remapped+peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 28498/250380 objects misplaced (11.382%); 30 MiB/s, 488 objects/s recovering 2024-08-17T17:47:14.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:14 smithi175 ceph-mon[101434]: 10.2 scrub starts 2024-08-17T17:47:14.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:14 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:14 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.736 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:14 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:47:14.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:14.865 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:14.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[104516]: pgmap v1437: 109 pgs: 1 activating, 2 peering, 1 remapped+peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 28498/250380 objects misplaced (11.382%); 30 MiB/s, 488 objects/s recovering 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[104516]: 10.2 scrub starts 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[109246]: pgmap v1437: 109 pgs: 1 activating, 2 peering, 1 remapped+peering, 6 active+remapped+backfill_wait, 3 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 28498/250380 objects misplaced (11.382%); 30 MiB/s, 488 objects/s recovering 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[109246]: 10.2 scrub starts 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:14.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:14 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:47:14.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:15.002 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:15.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:15.144 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:15.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.311 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:15.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:15.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:15.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:15.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:15 smithi064 ceph-mon[104516]: 10.2 scrub ok 2024-08-17T17:47:15.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:15 smithi064 ceph-mon[104516]: 10.10 scrub starts 2024-08-17T17:47:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:15 smithi064 ceph-mon[109246]: 10.2 scrub ok 2024-08-17T17:47:15.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:15 smithi064 ceph-mon[109246]: 10.10 scrub starts 2024-08-17T17:47:15.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.739 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:15.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.883 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:15.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:15.883 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:15 smithi175 ceph-mon[101434]: 10.2 scrub ok 2024-08-17T17:47:15.883 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:15 smithi175 ceph-mon[101434]: 10.10 scrub starts 2024-08-17T17:47:15.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:47:15.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:16.006 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:16.029 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:16.091 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:16.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:16.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:16.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:16.348 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:47:16.348 INFO:teuthology.orchestra.run.smithi175.stdout: "success": "" 2024-08-17T17:47:16.348 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:47:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.396 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:16.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:16.497 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:16.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:16.634 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.669 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:16.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:16.769 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:16.803 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:16.806 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:16 smithi175 ceph-mon[101434]: pgmap v1438: 109 pgs: 1 activating, 2 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 535 op/s; 33186/248505 objects misplaced (13.354%); 28 MiB/s, 455 objects/s recovering 2024-08-17T17:47:16.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:16 smithi064 ceph-mon[104516]: pgmap v1438: 109 pgs: 1 activating, 2 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 535 op/s; 33186/248505 objects misplaced (13.354%); 28 MiB/s, 455 objects/s recovering 2024-08-17T17:47:16.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:16 smithi064 ceph-mon[109246]: pgmap v1438: 109 pgs: 1 activating, 2 peering, 6 active+remapped+backfill_wait, 4 active+remapped+backfilling, 96 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 535 op/s; 33186/248505 objects misplaced (13.354%); 28 MiB/s, 455 objects/s recovering 2024-08-17T17:47:16.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:16.942 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:17.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:17.094 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:17.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:17.196 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:17.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:17.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:17.364 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:17.466 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:17.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:17.608 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:17.628 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:17.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.763 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:17.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:17.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:17.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:17.999 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:18.031 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:18.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:18.172 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:18.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:18.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:18.311 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:18.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:18.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:18.542 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:18.580 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:18.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:18.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:18.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:18.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:18.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:18.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:18 smithi175 ceph-mon[101434]: pgmap v1439: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.6 KiB/s rd, 0 B/s wr, 330 op/s; 28744/234309 objects misplaced (12.268%); 30 MiB/s, 482 objects/s recovering 2024-08-17T17:47:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:18 smithi064 ceph-mon[104516]: pgmap v1439: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.6 KiB/s rd, 0 B/s wr, 330 op/s; 28744/234309 objects misplaced (12.268%); 30 MiB/s, 482 objects/s recovering 2024-08-17T17:47:18.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:18 smithi064 ceph-mon[109246]: pgmap v1439: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.6 KiB/s rd, 0 B/s wr, 330 op/s; 28744/234309 objects misplaced (12.268%); 30 MiB/s, 482 objects/s recovering 2024-08-17T17:47:18.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:19.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:19.107 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:19.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:19.252 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.292 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:19.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:19.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:19.438 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:19.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:19.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.575 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:19.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:19.676 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:19.705 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:19.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:19.846 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:19.948 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:19.986 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:20.088 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:20.125 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:20.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:20.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:20.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:20.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:20.404 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:20.506 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:20.551 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:20.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:20.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:20.792 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:20.830 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:20 smithi175 ceph-mon[101434]: pgmap v1440: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 8.6 KiB/s rd, 0 B/s wr, 386 op/s; 27892/231873 objects misplaced (12.029%); 31 MiB/s, 492 objects/s recovering 2024-08-17T17:47:20.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:21.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:20 smithi064 ceph-mon[104516]: pgmap v1440: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 8.6 KiB/s rd, 0 B/s wr, 386 op/s; 27892/231873 objects misplaced (12.029%); 31 MiB/s, 492 objects/s recovering 2024-08-17T17:47:21.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:20 smithi064 ceph-mon[109246]: pgmap v1440: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 8.6 KiB/s rd, 0 B/s wr, 386 op/s; 27892/231873 objects misplaced (12.029%); 31 MiB/s, 492 objects/s recovering 2024-08-17T17:47:21.194 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:21.296 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.339 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:21.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:21.440 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:21.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:21.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:21.628 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:21.730 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:21.773 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:21.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:21.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:21 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:47:21.911 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:21.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:21 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:47:21.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:21 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:47:22.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:22.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.192 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:22.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:22.293 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.327 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:22.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:22.428 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:22.458 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:22.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:22.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:22.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:22.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:22.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:22.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:22.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:22 smithi175 ceph-mon[101434]: pgmap v1441: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 0 B/s wr, 445 op/s; 27756/225036 objects misplaced (12.334%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:47:22.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:22 smithi064 ceph-mon[104516]: pgmap v1441: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 0 B/s wr, 445 op/s; 27756/225036 objects misplaced (12.334%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:47:22.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:22 smithi064 ceph-mon[109246]: pgmap v1441: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 0 B/s wr, 445 op/s; 27756/225036 objects misplaced (12.334%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:47:22.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.015 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:23.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:23.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:23.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:23.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:23.289 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:23.298 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:47:23.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:23.298 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-08-17T17:47:23.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:47:23.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:23.458 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:23.560 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:23.593 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:23.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:23.832 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:23.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:23.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:23.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:24.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:24.110 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:24.150 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:24.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:24.235 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:23] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-08-17T17:47:24.249 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:24.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:24.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:24.422 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.456 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:24.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:24.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:24.592 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:24.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:24.735 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:24.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.882 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:24.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:24 smithi175 ceph-mon[101434]: 10.10 scrub starts 2024-08-17T17:47:24.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:24 smithi175 ceph-mon[101434]: pgmap v1442: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.4 KiB/s rd, 0 B/s wr, 554 op/s; 24052/218670 objects misplaced (10.999%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:47:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:24 smithi064 ceph-mon[104516]: 10.10 scrub starts 2024-08-17T17:47:24.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:24 smithi064 ceph-mon[104516]: pgmap v1442: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.4 KiB/s rd, 0 B/s wr, 554 op/s; 24052/218670 objects misplaced (10.999%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:47:24.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:24 smithi064 ceph-mon[109246]: 10.10 scrub starts 2024-08-17T17:47:24.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:24 smithi064 ceph-mon[109246]: pgmap v1442: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.4 KiB/s rd, 0 B/s wr, 554 op/s; 24052/218670 objects misplaced (10.999%); 20 MiB/s, 322 objects/s recovering 2024-08-17T17:47:24.984 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.025 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:25.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:25.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:25.163 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:25.265 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:25.306 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:25.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:25.408 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:25.441 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:25.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:25.585 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:25.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:25.729 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:25.831 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:25.869 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:25.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:25 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:47:25.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:25 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:47:25.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:25.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:25 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:47:25.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:25.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:26.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:26.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:26.391 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:26.425 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:26.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:26.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.569 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:26.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:26.712 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:26.814 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:26.852 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:26.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:26 smithi175 ceph-mon[101434]: 10.10 scrub starts 2024-08-17T17:47:26.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:26 smithi175 ceph-mon[101434]: pgmap v1443: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 513 op/s; 23148/216063 objects misplaced (10.714%); 19 MiB/s, 300 objects/s recovering 2024-08-17T17:47:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:26 smithi064 ceph-mon[104516]: 10.10 scrub starts 2024-08-17T17:47:26.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:26 smithi064 ceph-mon[104516]: pgmap v1443: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 513 op/s; 23148/216063 objects misplaced (10.714%); 19 MiB/s, 300 objects/s recovering 2024-08-17T17:47:26.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:26 smithi064 ceph-mon[109246]: 10.10 scrub starts 2024-08-17T17:47:26.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:26 smithi064 ceph-mon[109246]: pgmap v1443: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 513 op/s; 23148/216063 objects misplaced (10.714%); 19 MiB/s, 300 objects/s recovering 2024-08-17T17:47:26.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:26.988 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:26.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:27.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:27.130 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:27.232 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:27.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:27.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:27.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:27.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:27.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:27.680 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:27 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:47:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:27 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:47:27.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.843 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:27.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:27.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:27 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:47:27.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:27.968 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:28.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:28.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:28.254 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:28.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:28.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:28.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:28.547 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:28.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:28.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:47:28.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:47:28.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:28.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.701 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:28.802 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:28.831 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:28 smithi175 ceph-mon[101434]: pgmap v1444: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 21510/206049 objects misplaced (10.439%); 20 MiB/s, 317 objects/s recovering 2024-08-17T17:47:28.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1242731871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2299780778' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[104516]: pgmap v1444: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 21510/206049 objects misplaced (10.439%); 20 MiB/s, 317 objects/s recovering 2024-08-17T17:47:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1242731871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2299780778' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[109246]: pgmap v1444: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 21510/206049 objects misplaced (10.439%); 20 MiB/s, 317 objects/s recovering 2024-08-17T17:47:28.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1242731871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:28.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2299780778' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:28.974 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.113 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.331 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:47:29.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:29.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:29.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.548 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:29.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:29.710 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:29.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:29.848 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:29.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:47:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:47:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:29.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[104516]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:47:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:47:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:47:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:29.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:29 smithi064 ceph-mon[109246]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:47:29.949 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:29.959 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:29 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:47:29.959 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:29 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:47:29.959 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:29.959 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:29 smithi175 ceph-mon[101434]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:47:29.985 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:30.086 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.116 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:30.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:30.217 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:30.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:30.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:30.483 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:30.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:30.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:30.638 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:30.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:30.688 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:30.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.782 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:30.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:30.883 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:30.926 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:30.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:30.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:30 smithi064 ceph-mon[104516]: pgmap v1446: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18675/200361 objects misplaced (9.321%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:47:30.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:30 smithi064 ceph-mon[104516]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:47:30.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:30 smithi064 ceph-mon[109246]: pgmap v1446: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18675/200361 objects misplaced (9.321%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:47:30.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:30 smithi064 ceph-mon[109246]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:47:31.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:31.062 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:31.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:30 smithi175 ceph-mon[101434]: pgmap v1446: 109 pgs: 1 active+clean+scrubbing, 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 98 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18675/200361 objects misplaced (9.321%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:47:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:30 smithi175 ceph-mon[101434]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:47:31.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.201 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:31.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.341 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:31.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:31.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.494 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:31.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:31.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.633 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:31.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:31.769 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:31.871 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:31.902 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:31.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[104516]: 10.10 scrub starts 2024-08-17T17:47:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[104516]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:47:31.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[104516]: 10.18 deep-scrub starts 2024-08-17T17:47:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[109246]: 10.10 scrub starts 2024-08-17T17:47:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[109246]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:47:31.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:31 smithi064 ceph-mon[109246]: 10.18 deep-scrub starts 2024-08-17T17:47:32.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:32.033 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:32.135 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:32.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:31 smithi175 ceph-mon[101434]: 10.10 scrub starts 2024-08-17T17:47:32.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:31 smithi175 ceph-mon[101434]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:47:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:31 smithi175 ceph-mon[101434]: 10.18 deep-scrub starts 2024-08-17T17:47:32.167 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:32.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.308 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:32.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:32.449 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:32.551 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:32.581 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:32.683 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:32.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:47:32.718 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:32.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:32.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:32 smithi064 ceph-mon[104516]: pgmap v1449: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 18554/194313 objects misplaced (9.549%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:32.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:32 smithi064 ceph-mon[104516]: 10.13 scrub starts 2024-08-17T17:47:32.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:32 smithi064 ceph-mon[109246]: pgmap v1449: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 18554/194313 objects misplaced (9.549%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:32.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:32 smithi064 ceph-mon[109246]: 10.13 scrub starts 2024-08-17T17:47:32.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.874 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:32.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:32.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:33.003 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:33.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.159 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:33.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:32 smithi175 ceph-mon[101434]: pgmap v1449: 109 pgs: 7 active+remapped+backfill_wait, 3 active+remapped+backfilling, 99 active+clean; 4.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 18554/194313 objects misplaced (9.549%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:32 smithi175 ceph-mon[101434]: 10.13 scrub starts 2024-08-17T17:47:33.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:33.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:33.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:33.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:47:33.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:33.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:33.564 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:33.593 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:33.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:33.695 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:33.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.725 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:33.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:33.725 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:33.726 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[104516]: 10.13 scrub ok 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[104516]: 10.8 deep-scrub starts 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[104516]: 10.12 scrub starts 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[104516]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[109246]: 10.13 scrub ok 2024-08-17T17:47:33.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[109246]: 10.8 deep-scrub starts 2024-08-17T17:47:33.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[109246]: 10.12 scrub starts 2024-08-17T17:47:33.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-mon[109246]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:47:33.826 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:33.850 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:33.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:33.952 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:33.977 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:34.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:34.108 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:34.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:33 smithi175 ceph-mon[101434]: 10.13 scrub ok 2024-08-17T17:47:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:33 smithi175 ceph-mon[101434]: 10.8 deep-scrub starts 2024-08-17T17:47:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:33 smithi175 ceph-mon[101434]: 10.12 scrub starts 2024-08-17T17:47:34.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:33 smithi175 ceph-mon[101434]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:47:34.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:33] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.51.0" 2024-08-17T17:47:34.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:34.238 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:34.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:34.366 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:34.468 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:34.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:34.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:34.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:34.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:34.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:34.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:34.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:35.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:35.076 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:35.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:34 smithi175 ceph-mon[101434]: 10.6 scrub starts 2024-08-17T17:47:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:34 smithi175 ceph-mon[101434]: pgmap v1450: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15275/188874 objects misplaced (8.087%); 19 MiB/s, 308 objects/s recovering 2024-08-17T17:47:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:34 smithi175 ceph-mon[101434]: 10.6 scrub ok 2024-08-17T17:47:35.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:34 smithi175 ceph-mon[101434]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:47:35.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[109246]: 10.6 scrub starts 2024-08-17T17:47:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[109246]: pgmap v1450: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15275/188874 objects misplaced (8.087%); 19 MiB/s, 308 objects/s recovering 2024-08-17T17:47:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[109246]: 10.6 scrub ok 2024-08-17T17:47:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[109246]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:47:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[104516]: 10.6 scrub starts 2024-08-17T17:47:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[104516]: pgmap v1450: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15275/188874 objects misplaced (8.087%); 19 MiB/s, 308 objects/s recovering 2024-08-17T17:47:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[104516]: 10.6 scrub ok 2024-08-17T17:47:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:34 smithi064 ceph-mon[104516]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:47:35.214 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:35.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:35.492 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:35.594 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:35.625 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:35.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:47:35.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.756 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.858 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:35.890 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:47:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 10.1 scrub ok 2024-08-17T17:47:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 10.9 scrub starts 2024-08-17T17:47:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 10.9 scrub ok 2024-08-17T17:47:35.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 1.7 scrub starts 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[104516]: 1.7 scrub ok 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 10.1 scrub ok 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 10.9 scrub starts 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 10.9 scrub ok 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 1.7 scrub starts 2024-08-17T17:47:35.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:35 smithi064 ceph-mon[109246]: 1.7 scrub ok 2024-08-17T17:47:35.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.019 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:36.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:36.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.128 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:36.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:47:36.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 10.1 scrub ok 2024-08-17T17:47:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 10.9 scrub starts 2024-08-17T17:47:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 10.9 scrub ok 2024-08-17T17:47:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 1.7 scrub starts 2024-08-17T17:47:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:35 smithi175 ceph-mon[101434]: 1.7 scrub ok 2024-08-17T17:47:36.196 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:36.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:36.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.481 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:36.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:36.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.618 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:36.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:36.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:36.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:36.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.898 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:36.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:36.998 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.035 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:37.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:36 smithi175 ceph-mon[101434]: 10.10 scrub starts 2024-08-17T17:47:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:36 smithi175 ceph-mon[101434]: pgmap v1453: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14156/186912 objects misplaced (7.574%); 20 MiB/s, 315 objects/s recovering 2024-08-17T17:47:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:36 smithi175 ceph-mon[101434]: 10.10 scrub ok 2024-08-17T17:47:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:37.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[109246]: 10.10 scrub starts 2024-08-17T17:47:37.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[109246]: pgmap v1453: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14156/186912 objects misplaced (7.574%); 20 MiB/s, 315 objects/s recovering 2024-08-17T17:47:37.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[109246]: 10.10 scrub ok 2024-08-17T17:47:37.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[104516]: 10.10 scrub starts 2024-08-17T17:47:37.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[104516]: pgmap v1453: 109 pgs: 1 active+clean+scrubbing, 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 14156/186912 objects misplaced (7.574%); 20 MiB/s, 315 objects/s recovering 2024-08-17T17:47:37.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[104516]: 10.10 scrub ok 2024-08-17T17:47:37.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:37.287 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:37.291 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:37.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:37.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.436 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:37.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:37.537 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:37.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:37.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.826 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:37.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:37.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:37.951 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:38.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:38.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:38.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:38.178 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:38.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:38.328 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:38.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.466 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:38.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:38.566 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:38.598 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:38.700 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.728 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:38.830 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.859 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:38.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.987 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:39.089 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:39.115 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:39.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[104516]: pgmap v1454: 109 pgs: 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 1 active+remapped+backfilling, 99 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 13754/179112 objects misplaced (7.679%) 2024-08-17T17:47:39.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[104516]: 10.16 scrub starts 2024-08-17T17:47:39.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[104516]: 10.0 deep-scrub starts 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[104516]: 10.16 scrub ok 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[104516]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[109246]: pgmap v1454: 109 pgs: 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 1 active+remapped+backfilling, 99 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 13754/179112 objects misplaced (7.679%) 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[109246]: 10.16 scrub starts 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[109246]: 10.0 deep-scrub starts 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[109246]: 10.16 scrub ok 2024-08-17T17:47:39.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:38 smithi064 ceph-mon[109246]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:47:39.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:38 smithi175 ceph-mon[101434]: pgmap v1454: 109 pgs: 2 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 1 active+remapped+backfilling, 99 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 13754/179112 objects misplaced (7.679%) 2024-08-17T17:47:39.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:38 smithi175 ceph-mon[101434]: 10.16 scrub starts 2024-08-17T17:47:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:38 smithi175 ceph-mon[101434]: 10.0 deep-scrub starts 2024-08-17T17:47:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:38 smithi175 ceph-mon[101434]: 10.16 scrub ok 2024-08-17T17:47:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:38 smithi175 ceph-mon[101434]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:47:39.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:39.246 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:39.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:39.376 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:39.478 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:39.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:39.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:39.661 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:39.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:39.763 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:39.793 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:39.895 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:39.930 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:40.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:40.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:40.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:40.229 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:40.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:40.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:40.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:40.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.511 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:40.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:40.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:40.652 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:40.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:40.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[104516]: pgmap v1456: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 98 active+clean; 3.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 12311/175632 objects misplaced (7.010%) 2024-08-17T17:47:40.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[104516]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:47:40.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:40.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:40.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[109246]: pgmap v1456: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 98 active+clean; 3.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 12311/175632 objects misplaced (7.010%) 2024-08-17T17:47:40.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[109246]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:47:40.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:40.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:40.784 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:40.802 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:47:40.803 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:47:40.886 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:40 smithi175 ceph-mon[101434]: pgmap v1456: 109 pgs: 1 peering, 3 active+clean+scrubbing+deep, 7 active+remapped+backfill_wait, 98 active+clean; 3.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 12311/175632 objects misplaced (7.010%) 2024-08-17T17:47:40.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:40 smithi175 ceph-mon[101434]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:47:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:40.917 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:41.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:41.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.060 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:41.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:41.199 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:41.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.355 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.503 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:41.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:41.644 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:41.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:41.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:41 smithi064 ceph-mon[104516]: 10.19 deep-scrub starts 2024-08-17T17:47:41.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:41 smithi064 ceph-mon[104516]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:41.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:41 smithi064 ceph-mon[109246]: 10.19 deep-scrub starts 2024-08-17T17:47:41.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:41 smithi064 ceph-mon[109246]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:41.792 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:41 smithi175 ceph-mon[101434]: 10.19 deep-scrub starts 2024-08-17T17:47:41.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:41 smithi175 ceph-mon[101434]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:41.919 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1457,"stamp":"2024-08-17T17:47:39.505781+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3839650211,"num_objects":58544,"num_object_clones":0,"num_object_copies":175632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":12311,"num_objects_unfound":0,"num_objects_dirty":58544,"num_whiteouts":0,"num_read":6023,"num_read_kb":13076,"num_write":157422,"num_write_kb":6510030,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9707,"num_bytes_recovered":635671240,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":218320,"ondisk_log_size":218320,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19517840,"kb_used_data":12531764,"kb_used_omap":46,"kb_used_meta":6985745,"kb_avail":730443376,"statfs":{"total":767960285184,"available":747974017024,"internally_reserved":0,"allocated":12832526336,"data_stored":12814706190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":7153403123},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-289275904,"num_objects":-4414,"num_object_clones":0,"num_object_copies":-13242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2964,"num_objects_unfound":0,"num_objects_dirty":-4414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-25201,"num_write_kb":-1251136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3051,"num_bytes_recovered":-199884800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.016593"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489532+0000","last_change":"2024-08-17T17:47:08.499597+0000","last_active":"2024-08-17T17:47:34.489532+0000","last_peered":"2024-08-17T17:47:34.489532+0000","last_clean":"2024-08-17T17:47:34.489532+0000","last_became_active":"2024-08-17T17:47:08.499282+0000","last_became_peered":"2024-08-17T17:47:08.499282+0000","last_unstale":"2024-08-17T17:47:34.489532+0000","last_undegraded":"2024-08-17T17:47:34.489532+0000","last_fullsized":"2024-08-17T17:47:34.489532+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:03.581238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":458,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.494468+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:47:34.494468+0000","last_peered":"2024-08-17T17:47:34.494468+0000","last_clean":"2024-08-17T17:47:34.494468+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:47:34.494468+0000","last_undegraded":"2024-08-17T17:47:34.494468+0000","last_fullsized":"2024-08-17T17:47:34.494468+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":258,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.489045+0000","last_change":"2024-08-17T17:47:08.492972+0000","last_active":"2024-08-17T17:47:33.489045+0000","last_peered":"2024-08-17T17:47:33.489045+0000","last_clean":"2024-08-17T17:47:33.489045+0000","last_became_active":"2024-08-17T17:47:08.492623+0000","last_became_peered":"2024-08-17T17:47:08.492623+0000","last_unstale":"2024-08-17T17:47:33.489045+0000","last_undegraded":"2024-08-17T17:47:33.489045+0000","last_fullsized":"2024-08-17T17:47:33.489045+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:57.182152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":778,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497015+0000","last_change":"2024-08-17T17:47:08.483488+0000","last_active":"2024-08-17T17:47:34.497015+0000","last_peered":"2024-08-17T17:47:34.497015+0000","last_clean":"2024-08-17T17:47:34.497015+0000","last_became_active":"2024-08-17T17:47:08.483171+0000","last_became_peered":"2024-08-17T17:47:08.483171+0000","last_unstale":"2024-08-17T17:47:34.497015+0000","last_undegraded":"2024-08-17T17:47:34.497015+0000","last_fullsized":"2024-08-17T17:47:34.497015+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:19.349184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"455'11436","reported_seq":11602,"reported_epoch":455,"state":"active+clean+scrubbing+deep","last_fresh":"2024-08-17T17:47:37.547082+0000","last_change":"2024-08-17T17:47:31.305673+0000","last_active":"2024-08-17T17:47:37.547082+0000","last_peered":"2024-08-17T17:47:37.547082+0000","last_clean":"2024-08-17T17:47:37.547082+0000","last_became_active":"2024-08-17T17:46:52.736037+0000","last_became_peered":"2024-08-17T17:46:52.736037+0000","last_unstale":"2024-08-17T17:47:37.547082+0000","last_undegraded":"2024-08-17T17:47:37.547082+0000","last_fullsized":"2024-08-17T17:47:37.547082+0000","mapping_epoch":437,"log_start":"420'2000","ondisk_log_start":"420'2000","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":836,"log_size":9436,"log_dups_size":0,"ondisk_log_size":9436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 6s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118128640,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5911,"num_write_kb":246816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":778,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497427+0000","last_change":"2024-08-17T17:47:08.492303+0000","last_active":"2024-08-17T17:47:34.497427+0000","last_peered":"2024-08-17T17:47:34.497427+0000","last_clean":"2024-08-17T17:47:34.497427+0000","last_became_active":"2024-08-17T17:47:08.491977+0000","last_became_peered":"2024-08-17T17:47:08.491977+0000","last_unstale":"2024-08-17T17:47:34.497427+0000","last_undegraded":"2024-08-17T17:47:34.497427+0000","last_fullsized":"2024-08-17T17:47:34.497427+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:15.963818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":444,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.495179+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:47:34.495179+0000","last_peered":"2024-08-17T17:47:34.495179+0000","last_clean":"2024-08-17T17:47:34.495179+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:47:34.495179+0000","last_undegraded":"2024-08-17T17:47:34.495179+0000","last_fullsized":"2024-08-17T17:47:34.495179+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.19","version":"455'10717","reported_seq":10782,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.655489+0000","last_change":"2024-08-17T17:46:52.802372+0000","last_active":"2024-08-17T17:47:35.655489+0000","last_peered":"2024-08-17T17:47:35.655489+0000","last_clean":"2024-08-17T17:47:35.655489+0000","last_became_active":"2024-08-17T17:46:52.776987+0000","last_became_peered":"2024-08-17T17:46:52.776987+0000","last_unstale":"2024-08-17T17:47:35.655489+0000","last_undegraded":"2024-08-17T17:47:35.655489+0000","last_fullsized":"2024-08-17T17:47:35.655489+0000","mapping_epoch":437,"log_start":"419'1300","ondisk_log_start":"419'1300","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9417,"log_dups_size":0,"ondisk_log_size":9417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121536512,"num_objects":1855,"num_object_clones":0,"num_object_copies":5565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":250432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489989+0000","last_change":"2024-08-17T17:47:08.498174+0000","last_active":"2024-08-17T17:47:34.489989+0000","last_peered":"2024-08-17T17:47:34.489989+0000","last_clean":"2024-08-17T17:47:34.489989+0000","last_became_active":"2024-08-17T17:47:08.497836+0000","last_became_peered":"2024-08-17T17:47:08.497836+0000","last_unstale":"2024-08-17T17:47:34.489989+0000","last_undegraded":"2024-08-17T17:47:34.489989+0000","last_fullsized":"2024-08-17T17:47:34.489989+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:09.116261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":392,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.487323+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:47:34.487323+0000","last_peered":"2024-08-17T17:47:34.487323+0000","last_clean":"2024-08-17T17:47:34.487323+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:47:34.487323+0000","last_undegraded":"2024-08-17T17:47:34.487323+0000","last_fullsized":"2024-08-17T17:47:34.487323+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"455'10712","reported_seq":10827,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.604261+0000","last_change":"2024-08-17T17:47:33.743471+0000","last_active":"2024-08-17T17:47:35.604261+0000","last_peered":"2024-08-17T17:47:35.604261+0000","last_clean":"2024-08-17T17:47:35.604261+0000","last_became_active":"2024-08-17T17:46:52.702751+0000","last_became_peered":"2024-08-17T17:46:52.702751+0000","last_unstale":"2024-08-17T17:47:35.604261+0000","last_undegraded":"2024-08-17T17:47:35.604261+0000","last_fullsized":"2024-08-17T17:47:35.604261+0000","mapping_epoch":410,"log_start":"419'1400","ondisk_log_start":"419'1400","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'10681","last_scrub_stamp":"2024-08-17T17:47:33.743333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:33.743333+0000","objects_scrubbed":1909,"log_size":9312,"log_dups_size":0,"ondisk_log_size":9312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:19.393314+0000","scrub_duration":0.34479844399999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":453,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491551+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:47:33.491551+0000","last_peered":"2024-08-17T17:47:33.491551+0000","last_clean":"2024-08-17T17:47:33.491551+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:47:33.491551+0000","last_undegraded":"2024-08-17T17:47:33.491551+0000","last_fullsized":"2024-08-17T17:47:33.491551+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488039+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:47:33.488039+0000","last_peered":"2024-08-17T17:47:33.488039+0000","last_clean":"2024-08-17T17:47:33.488039+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:47:33.488039+0000","last_undegraded":"2024-08-17T17:47:33.488039+0000","last_fullsized":"2024-08-17T17:47:33.488039+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"455'10675","reported_seq":10751,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:36.635233+0000","last_change":"2024-08-17T17:47:09.947034+0000","last_active":"2024-08-17T17:47:36.635233+0000","last_peered":"2024-08-17T17:47:36.635233+0000","last_clean":"2024-08-17T17:47:07.399347+0000","last_became_active":"2024-08-17T17:47:09.926064+0000","last_became_peered":"2024-08-17T17:47:09.926064+0000","last_unstale":"2024-08-17T17:47:36.635233+0000","last_undegraded":"2024-08-17T17:47:36.635233+0000","last_fullsized":"2024-08-17T17:47:36.635233+0000","mapping_epoch":447,"log_start":"418'1200","ondisk_log_start":"418'1200","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9475,"log_dups_size":0,"ondisk_log_size":9475,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1721,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":246272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1721}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":454,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491262+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:47:33.491262+0000","last_peered":"2024-08-17T17:47:33.491262+0000","last_clean":"2024-08-17T17:47:33.491262+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:47:33.491262+0000","last_undegraded":"2024-08-17T17:47:33.491262+0000","last_fullsized":"2024-08-17T17:47:33.491262+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":367,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497219+0000","last_change":"2024-08-17T17:47:08.489953+0000","last_active":"2024-08-17T17:47:33.497219+0000","last_peered":"2024-08-17T17:47:33.497219+0000","last_clean":"2024-08-17T17:47:33.497219+0000","last_became_active":"2024-08-17T17:47:08.489588+0000","last_became_peered":"2024-08-17T17:47:08.489588+0000","last_unstale":"2024-08-17T17:47:33.497219+0000","last_undegraded":"2024-08-17T17:47:33.497219+0000","last_fullsized":"2024-08-17T17:47:33.497219+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:33.896317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"455'10669","reported_seq":10798,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:37.494619+0000","last_change":"2024-08-17T17:47:09.847119+0000","last_active":"2024-08-17T17:47:37.494619+0000","last_peered":"2024-08-17T17:47:37.494619+0000","last_clean":"2024-08-17T17:47:07.502206+0000","last_became_active":"2024-08-17T17:47:09.819565+0000","last_became_peered":"2024-08-17T17:47:09.819565+0000","last_unstale":"2024-08-17T17:47:37.494619+0000","last_undegraded":"2024-08-17T17:47:37.494619+0000","last_fullsized":"2024-08-17T17:47:37.494619+0000","mapping_epoch":447,"log_start":"420'2100","ondisk_log_start":"420'2100","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8569,"log_dups_size":0,"ondisk_log_size":8569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122290176,"num_objects":1866,"num_object_clones":0,"num_object_copies":5598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1866,"num_objects_unfound":0,"num_objects_dirty":1866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5927,"num_write_kb":249376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1866}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":471,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491429+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:47:33.491429+0000","last_peered":"2024-08-17T17:47:33.491429+0000","last_clean":"2024-08-17T17:47:33.491429+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:47:33.491429+0000","last_undegraded":"2024-08-17T17:47:33.491429+0000","last_fullsized":"2024-08-17T17:47:33.491429+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":373,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.488424+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:47:34.488424+0000","last_peered":"2024-08-17T17:47:34.488424+0000","last_clean":"2024-08-17T17:47:34.488424+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:47:34.488424+0000","last_undegraded":"2024-08-17T17:47:34.488424+0000","last_fullsized":"2024-08-17T17:47:34.488424+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"455'10720","reported_seq":10850,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:37.521624+0000","last_change":"2024-08-17T17:47:09.829436+0000","last_active":"2024-08-17T17:47:37.521624+0000","last_peered":"2024-08-17T17:47:37.521624+0000","last_clean":"2024-08-17T17:47:07.467949+0000","last_became_active":"2024-08-17T17:47:09.805225+0000","last_became_peered":"2024-08-17T17:47:09.805225+0000","last_unstale":"2024-08-17T17:47:37.521624+0000","last_undegraded":"2024-08-17T17:47:37.521624+0000","last_fullsized":"2024-08-17T17:47:37.521624+0000","mapping_epoch":447,"log_start":"420'2100","ondisk_log_start":"420'2100","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8620,"log_dups_size":0,"ondisk_log_size":8620,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121569280,"num_objects":1855,"num_object_clones":0,"num_object_copies":5565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1855,"num_objects_unfound":0,"num_objects_dirty":1855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":250464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1855}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":269,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488713+0000","last_change":"2024-08-17T17:47:08.490413+0000","last_active":"2024-08-17T17:47:33.488713+0000","last_peered":"2024-08-17T17:47:33.488713+0000","last_clean":"2024-08-17T17:47:33.488713+0000","last_became_active":"2024-08-17T17:47:08.489925+0000","last_became_peered":"2024-08-17T17:47:08.489925+0000","last_unstale":"2024-08-17T17:47:33.488713+0000","last_undegraded":"2024-08-17T17:47:33.488713+0000","last_fullsized":"2024-08-17T17:47:33.488713+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:39.150961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":786,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497172+0000","last_change":"2024-08-17T17:45:27.582532+0000","last_active":"2024-08-17T17:47:34.497172+0000","last_peered":"2024-08-17T17:47:34.497172+0000","last_clean":"2024-08-17T17:47:34.497172+0000","last_became_active":"2024-08-17T17:45:27.582184+0000","last_became_peered":"2024-08-17T17:45:27.582184+0000","last_unstale":"2024-08-17T17:47:34.497172+0000","last_undegraded":"2024-08-17T17:47:34.497172+0000","last_fullsized":"2024-08-17T17:47:34.497172+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:28.064920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"455'10671","reported_seq":10827,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.592614+0000","last_change":"2024-08-17T17:47:14.318183+0000","last_active":"2024-08-17T17:47:36.592614+0000","last_peered":"2024-08-17T17:47:36.592614+0000","last_clean":"2024-08-17T17:47:36.592614+0000","last_became_active":"2024-08-17T17:46:52.725691+0000","last_became_peered":"2024-08-17T17:46:52.725691+0000","last_unstale":"2024-08-17T17:47:36.592614+0000","last_undegraded":"2024-08-17T17:47:36.592614+0000","last_fullsized":"2024-08-17T17:47:36.592614+0000","mapping_epoch":410,"log_start":"420'2600","ondisk_log_start":"420'2600","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"450'10043","last_scrub_stamp":"2024-08-17T17:47:14.317813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:14.317813+0000","objects_scrubbed":2488,"log_size":8071,"log_dups_size":0,"ondisk_log_size":8071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:46.208883+0000","scrub_duration":875,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121110528,"num_objects":1848,"num_object_clones":0,"num_object_copies":5544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":640,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":846,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.490465+0000","last_change":"2024-08-17T17:47:11.537662+0000","last_active":"2024-08-17T17:47:33.490465+0000","last_peered":"2024-08-17T17:47:33.490465+0000","last_clean":"2024-08-17T17:47:33.490465+0000","last_became_active":"2024-08-17T17:47:11.536791+0000","last_became_peered":"2024-08-17T17:47:11.536791+0000","last_unstale":"2024-08-17T17:47:33.490465+0000","last_undegraded":"2024-08-17T17:47:33.490465+0000","last_fullsized":"2024-08-17T17:47:33.490465+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:54.176207+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":32,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489232+0000","last_change":"2024-08-17T17:47:08.509122+0000","last_active":"2024-08-17T17:47:34.489232+0000","last_peered":"2024-08-17T17:47:34.489232+0000","last_clean":"2024-08-17T17:47:34.489232+0000","last_became_active":"2024-08-17T17:47:08.497368+0000","last_became_peered":"2024-08-17T17:47:08.497368+0000","last_unstale":"2024-08-17T17:47:34.489232+0000","last_undegraded":"2024-08-17T17:47:34.489232+0000","last_fullsized":"2024-08-17T17:47:34.489232+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:35:01.638171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":391,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497850+0000","last_change":"2024-08-17T17:47:08.493553+0000","last_active":"2024-08-17T17:47:33.497850+0000","last_peered":"2024-08-17T17:47:33.497850+0000","last_clean":"2024-08-17T17:47:33.497850+0000","last_became_active":"2024-08-17T17:47:08.493193+0000","last_became_peered":"2024-08-17T17:47:08.493193+0000","last_unstale":"2024-08-17T17:47:33.497850+0000","last_undegraded":"2024-08-17T17:47:33.497850+0000","last_fullsized":"2024-08-17T17:47:33.497850+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:10.520928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"455'10747","reported_seq":11489,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:35.688942+0000","last_change":"2024-08-17T17:47:11.537223+0000","last_active":"2024-08-17T17:47:35.688942+0000","last_peered":"2024-08-17T17:47:35.688942+0000","last_clean":"2024-08-17T17:47:07.484811+0000","last_became_active":"2024-08-17T17:47:09.816596+0000","last_became_peered":"2024-08-17T17:47:09.816596+0000","last_unstale":"2024-08-17T17:47:35.688942+0000","last_undegraded":"2024-08-17T17:47:35.688942+0000","last_fullsized":"2024-08-17T17:47:35.688942+0000","mapping_epoch":447,"log_start":"419'1300","ondisk_log_start":"419'1300","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9447,"log_dups_size":0,"ondisk_log_size":9447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117276672,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1565,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6000,"num_write_kb":249280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":329,"num_bytes_recovered":21561344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,4],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1789}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":514,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.487957+0000","last_change":"2024-08-17T17:47:11.534000+0000","last_active":"2024-08-17T17:47:33.487957+0000","last_peered":"2024-08-17T17:47:33.487957+0000","last_clean":"2024-08-17T17:47:33.487957+0000","last_became_active":"2024-08-17T17:47:11.533623+0000","last_became_peered":"2024-08-17T17:47:11.533623+0000","last_unstale":"2024-08-17T17:47:33.487957+0000","last_undegraded":"2024-08-17T17:47:33.487957+0000","last_fullsized":"2024-08-17T17:47:33.487957+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:28.627072+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":379,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497179+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:47:33.497179+0000","last_peered":"2024-08-17T17:47:33.497179+0000","last_clean":"2024-08-17T17:47:33.497179+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:47:33.497179+0000","last_undegraded":"2024-08-17T17:47:33.497179+0000","last_fullsized":"2024-08-17T17:47:33.497179+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":349,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.498778+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:47:33.498778+0000","last_peered":"2024-08-17T17:47:33.498778+0000","last_clean":"2024-08-17T17:47:33.498778+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:47:33.498778+0000","last_undegraded":"2024-08-17T17:47:33.498778+0000","last_fullsized":"2024-08-17T17:47:33.498778+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"456'11346","reported_seq":11440,"reported_epoch":456,"state":"active+clean+scrubbing+deep","last_fresh":"2024-08-17T17:47:39.263910+0000","last_change":"2024-08-17T17:47:37.843881+0000","last_active":"2024-08-17T17:47:39.263910+0000","last_peered":"2024-08-17T17:47:39.263910+0000","last_clean":"2024-08-17T17:47:39.263910+0000","last_became_active":"2024-08-17T17:46:52.738619+0000","last_became_peered":"2024-08-17T17:46:52.738619+0000","last_unstale":"2024-08-17T17:47:39.263910+0000","last_undegraded":"2024-08-17T17:47:39.263910+0000","last_fullsized":"2024-08-17T17:47:39.263910+0000","mapping_epoch":410,"log_start":"420'3200","ondisk_log_start":"420'3200","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":524,"log_size":8146,"log_dups_size":0,"ondisk_log_size":8146,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 1s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119570432,"num_objects":1824,"num_object_clones":0,"num_object_copies":5472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6049,"num_write_kb":251968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":349,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497085+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:47:33.497085+0000","last_peered":"2024-08-17T17:47:33.497085+0000","last_clean":"2024-08-17T17:47:33.497085+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:47:33.497085+0000","last_undegraded":"2024-08-17T17:47:33.497085+0000","last_fullsized":"2024-08-17T17:47:33.497085+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":364,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.498253+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:47:33.498253+0000","last_peered":"2024-08-17T17:47:33.498253+0000","last_clean":"2024-08-17T17:47:33.498253+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:47:33.498253+0000","last_undegraded":"2024-08-17T17:47:33.498253+0000","last_fullsized":"2024-08-17T17:47:33.498253+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"455'10588","reported_seq":10704,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.093919+0000","last_change":"2024-08-17T17:47:34.436571+0000","last_active":"2024-08-17T17:47:36.093919+0000","last_peered":"2024-08-17T17:47:36.093919+0000","last_clean":"2024-08-17T17:47:36.093919+0000","last_became_active":"2024-08-17T17:46:52.726019+0000","last_became_peered":"2024-08-17T17:46:52.726019+0000","last_unstale":"2024-08-17T17:47:36.093919+0000","last_undegraded":"2024-08-17T17:47:36.093919+0000","last_fullsized":"2024-08-17T17:47:36.093919+0000","mapping_epoch":410,"log_start":"420'3200","ondisk_log_start":"420'3200","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'10553","last_scrub_stamp":"2024-08-17T17:47:34.436437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:34.436437+0000","objects_scrubbed":1896,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:51.170531+0000","scrub_duration":0.22765941100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121962496,"num_objects":1861,"num_object_clones":0,"num_object_copies":5583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":35,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":472,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.489384+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:47:33.489384+0000","last_peered":"2024-08-17T17:47:33.489384+0000","last_clean":"2024-08-17T17:47:33.489384+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:47:33.489384+0000","last_undegraded":"2024-08-17T17:47:33.489384+0000","last_fullsized":"2024-08-17T17:47:33.489384+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.488944+0000","last_change":"2024-08-17T17:47:08.498246+0000","last_active":"2024-08-17T17:47:34.488944+0000","last_peered":"2024-08-17T17:47:34.488944+0000","last_clean":"2024-08-17T17:47:34.488944+0000","last_became_active":"2024-08-17T17:47:08.497969+0000","last_became_peered":"2024-08-17T17:47:08.497969+0000","last_unstale":"2024-08-17T17:47:34.488944+0000","last_undegraded":"2024-08-17T17:47:34.488944+0000","last_fullsized":"2024-08-17T17:47:34.488944+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:39.344214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":776,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.488617+0000","last_change":"2024-08-17T17:47:08.493415+0000","last_active":"2024-08-17T17:47:34.488617+0000","last_peered":"2024-08-17T17:47:34.488617+0000","last_clean":"2024-08-17T17:47:34.488617+0000","last_became_active":"2024-08-17T17:47:08.493074+0000","last_became_peered":"2024-08-17T17:47:08.493074+0000","last_unstale":"2024-08-17T17:47:34.488617+0000","last_undegraded":"2024-08-17T17:47:34.488617+0000","last_fullsized":"2024-08-17T17:47:34.488617+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:39.768357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"455'11904","reported_seq":12012,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.658451+0000","last_change":"2024-08-17T17:44:36.610395+0000","last_active":"2024-08-17T17:47:36.658451+0000","last_peered":"2024-08-17T17:47:36.658451+0000","last_clean":"2024-08-17T17:47:36.658451+0000","last_became_active":"2024-08-17T17:44:36.610060+0000","last_became_peered":"2024-08-17T17:44:36.610060+0000","last_unstale":"2024-08-17T17:47:36.658451+0000","last_undegraded":"2024-08-17T17:47:36.658451+0000","last_fullsized":"2024-08-17T17:47:36.658451+0000","mapping_epoch":410,"log_start":"425'4324","ondisk_log_start":"425'4324","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:28.103243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11904,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":796,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.503304+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:47:33.503304+0000","last_peered":"2024-08-17T17:47:33.503304+0000","last_clean":"2024-08-17T17:47:33.503304+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:47:33.503304+0000","last_undegraded":"2024-08-17T17:47:33.503304+0000","last_fullsized":"2024-08-17T17:47:33.503304+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1049,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488412+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:47:33.488412+0000","last_peered":"2024-08-17T17:47:33.488412+0000","last_clean":"2024-08-17T17:47:33.488412+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:47:33.488412+0000","last_undegraded":"2024-08-17T17:47:33.488412+0000","last_fullsized":"2024-08-17T17:47:33.488412+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489225+0000","last_change":"2024-08-17T17:47:08.494090+0000","last_active":"2024-08-17T17:47:34.489225+0000","last_peered":"2024-08-17T17:47:34.489225+0000","last_clean":"2024-08-17T17:47:34.489225+0000","last_became_active":"2024-08-17T17:47:08.493780+0000","last_became_peered":"2024-08-17T17:47:08.493780+0000","last_unstale":"2024-08-17T17:47:34.489225+0000","last_undegraded":"2024-08-17T17:47:34.489225+0000","last_fullsized":"2024-08-17T17:47:34.489225+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:01:11.668902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":363,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.498664+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:47:33.498664+0000","last_peered":"2024-08-17T17:47:33.498664+0000","last_clean":"2024-08-17T17:47:33.498664+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:47:33.498664+0000","last_undegraded":"2024-08-17T17:47:33.498664+0000","last_fullsized":"2024-08-17T17:47:33.498664+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"455'11323","reported_seq":11468,"reported_epoch":455,"state":"active+clean+scrubbing+deep","last_fresh":"2024-08-17T17:47:37.549244+0000","last_change":"2024-08-17T17:47:32.352142+0000","last_active":"2024-08-17T17:47:37.549244+0000","last_peered":"2024-08-17T17:47:37.549244+0000","last_clean":"2024-08-17T17:47:37.549244+0000","last_became_active":"2024-08-17T17:46:52.709277+0000","last_became_peered":"2024-08-17T17:46:52.709277+0000","last_unstale":"2024-08-17T17:47:37.549244+0000","last_undegraded":"2024-08-17T17:47:37.549244+0000","last_fullsized":"2024-08-17T17:47:37.549244+0000","mapping_epoch":410,"log_start":"420'2000","ondisk_log_start":"420'2000","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":674,"log_size":9323,"log_dups_size":0,"ondisk_log_size":9323,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 5s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114917376,"num_objects":1753,"num_object_clones":0,"num_object_copies":5259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5959,"num_write_kb":246816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":575,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485428+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:47:34.485428+0000","last_peered":"2024-08-17T17:47:34.485428+0000","last_clean":"2024-08-17T17:47:34.485428+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:47:34.485428+0000","last_undegraded":"2024-08-17T17:47:34.485428+0000","last_fullsized":"2024-08-17T17:47:34.485428+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":837,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.503236+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:47:33.503236+0000","last_peered":"2024-08-17T17:47:33.503236+0000","last_clean":"2024-08-17T17:47:33.503236+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:47:33.503236+0000","last_undegraded":"2024-08-17T17:47:33.503236+0000","last_fullsized":"2024-08-17T17:47:33.503236+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":794,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.496857+0000","last_change":"2024-08-17T17:47:08.495040+0000","last_active":"2024-08-17T17:47:34.496857+0000","last_peered":"2024-08-17T17:47:34.496857+0000","last_clean":"2024-08-17T17:47:34.496857+0000","last_became_active":"2024-08-17T17:47:08.494475+0000","last_became_peered":"2024-08-17T17:47:08.494475+0000","last_unstale":"2024-08-17T17:47:34.496857+0000","last_undegraded":"2024-08-17T17:47:34.496857+0000","last_fullsized":"2024-08-17T17:47:34.496857+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:17.169290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":777,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497691+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:47:34.497691+0000","last_peered":"2024-08-17T17:47:34.497691+0000","last_clean":"2024-08-17T17:47:34.497691+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:47:34.497691+0000","last_undegraded":"2024-08-17T17:47:34.497691+0000","last_fullsized":"2024-08-17T17:47:34.497691+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"455'10690","reported_seq":10803,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.681048+0000","last_change":"2024-08-17T17:47:34.775376+0000","last_active":"2024-08-17T17:47:35.681048+0000","last_peered":"2024-08-17T17:47:35.681048+0000","last_clean":"2024-08-17T17:47:35.681048+0000","last_became_active":"2024-08-17T17:46:52.751923+0000","last_became_peered":"2024-08-17T17:46:52.751923+0000","last_unstale":"2024-08-17T17:47:35.681048+0000","last_undegraded":"2024-08-17T17:47:35.681048+0000","last_fullsized":"2024-08-17T17:47:35.681048+0000","mapping_epoch":410,"log_start":"419'1300","ondisk_log_start":"419'1300","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'10662","last_scrub_stamp":"2024-08-17T17:47:34.775143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:34.775143+0000","objects_scrubbed":1878,"log_size":9390,"log_dups_size":0,"ondisk_log_size":9390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:46.436810+0000","scrub_duration":0.33779308600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121110528,"num_objects":1848,"num_object_clones":0,"num_object_copies":5544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":521,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.490022+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:47:33.490022+0000","last_peered":"2024-08-17T17:47:33.490022+0000","last_clean":"2024-08-17T17:47:33.490022+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:47:33.490022+0000","last_undegraded":"2024-08-17T17:47:33.490022+0000","last_fullsized":"2024-08-17T17:47:33.490022+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":50,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489145+0000","last_change":"2024-08-17T17:47:08.519932+0000","last_active":"2024-08-17T17:47:34.489145+0000","last_peered":"2024-08-17T17:47:34.489145+0000","last_clean":"2024-08-17T17:47:34.489145+0000","last_became_active":"2024-08-17T17:47:08.499007+0000","last_became_peered":"2024-08-17T17:47:08.499007+0000","last_unstale":"2024-08-17T17:47:34.489145+0000","last_undegraded":"2024-08-17T17:47:34.489145+0000","last_fullsized":"2024-08-17T17:47:34.489145+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:09.837850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":769,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485061+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:47:34.485061+0000","last_peered":"2024-08-17T17:47:34.485061+0000","last_clean":"2024-08-17T17:47:34.485061+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:47:34.485061+0000","last_undegraded":"2024-08-17T17:47:34.485061+0000","last_fullsized":"2024-08-17T17:47:34.485061+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":816,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485599+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:47:34.485599+0000","last_peered":"2024-08-17T17:47:34.485599+0000","last_clean":"2024-08-17T17:47:34.485599+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:47:34.485599+0000","last_undegraded":"2024-08-17T17:47:34.485599+0000","last_fullsized":"2024-08-17T17:47:34.485599+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"456'12207","reported_seq":12263,"reported_epoch":456,"state":"active+clean","last_fresh":"2024-08-17T17:47:38.782474+0000","last_change":"2024-08-17T17:44:36.610536+0000","last_active":"2024-08-17T17:47:38.782474+0000","last_peered":"2024-08-17T17:47:38.782474+0000","last_clean":"2024-08-17T17:47:38.782474+0000","last_became_active":"2024-08-17T17:44:36.610195+0000","last_became_peered":"2024-08-17T17:44:36.610195+0000","last_unstale":"2024-08-17T17:47:38.782474+0000","last_undegraded":"2024-08-17T17:47:38.782474+0000","last_fullsized":"2024-08-17T17:47:38.782474+0000","mapping_epoch":410,"log_start":"426'5027","ondisk_log_start":"426'5027","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7180,"log_dups_size":0,"ondisk_log_size":7180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:16.686185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237830144,"num_objects":3629,"num_object_clones":0,"num_object_copies":10887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12207,"num_write_kb":506752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":472,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.489698+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:47:33.489698+0000","last_peered":"2024-08-17T17:47:33.489698+0000","last_clean":"2024-08-17T17:47:33.489698+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:47:33.489698+0000","last_undegraded":"2024-08-17T17:47:33.489698+0000","last_fullsized":"2024-08-17T17:47:33.489698+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1172,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.489571+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:47:33.489571+0000","last_peered":"2024-08-17T17:47:33.489571+0000","last_clean":"2024-08-17T17:47:33.489571+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:47:33.489571+0000","last_undegraded":"2024-08-17T17:47:33.489571+0000","last_fullsized":"2024-08-17T17:47:33.489571+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":595,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.487124+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:47:34.487124+0000","last_peered":"2024-08-17T17:47:34.487124+0000","last_clean":"2024-08-17T17:47:34.487124+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:47:34.487124+0000","last_undegraded":"2024-08-17T17:47:34.487124+0000","last_fullsized":"2024-08-17T17:47:34.487124+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":121,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488376+0000","last_change":"2024-08-17T17:47:08.490132+0000","last_active":"2024-08-17T17:47:33.488376+0000","last_peered":"2024-08-17T17:47:33.488376+0000","last_clean":"2024-08-17T17:47:33.488376+0000","last_became_active":"2024-08-17T17:47:08.489723+0000","last_became_peered":"2024-08-17T17:47:08.489723+0000","last_unstale":"2024-08-17T17:47:33.488376+0000","last_undegraded":"2024-08-17T17:47:33.488376+0000","last_fullsized":"2024-08-17T17:47:33.488376+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:30.591521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"455'12037","reported_seq":12092,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.073858+0000","last_change":"2024-08-17T17:44:36.607785+0000","last_active":"2024-08-17T17:47:36.073858+0000","last_peered":"2024-08-17T17:47:36.073858+0000","last_clean":"2024-08-17T17:47:36.073858+0000","last_became_active":"2024-08-17T17:44:36.607472+0000","last_became_peered":"2024-08-17T17:44:36.607472+0000","last_unstale":"2024-08-17T17:47:36.073858+0000","last_undegraded":"2024-08-17T17:47:36.073858+0000","last_fullsized":"2024-08-17T17:47:36.073858+0000","mapping_epoch":410,"log_start":"428'5351","ondisk_log_start":"428'5351","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:39.734707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240451584,"num_objects":3669,"num_object_clones":0,"num_object_copies":11007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12037,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":728,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.490055+0000","last_change":"2024-08-17T17:47:30.671421+0000","last_active":"2024-08-17T17:47:34.490055+0000","last_peered":"2024-08-17T17:47:34.490055+0000","last_clean":"2024-08-17T17:47:34.490055+0000","last_became_active":"2024-08-17T17:47:11.534662+0000","last_became_peered":"2024-08-17T17:47:11.534662+0000","last_unstale":"2024-08-17T17:47:34.490055+0000","last_undegraded":"2024-08-17T17:47:34.490055+0000","last_fullsized":"2024-08-17T17:47:34.490055+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:44:52.091249+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:47:35.012768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":259,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488857+0000","last_change":"2024-08-17T17:47:08.492880+0000","last_active":"2024-08-17T17:47:33.488857+0000","last_peered":"2024-08-17T17:47:33.488857+0000","last_clean":"2024-08-17T17:47:33.488857+0000","last_became_active":"2024-08-17T17:47:08.492557+0000","last_became_peered":"2024-08-17T17:47:08.492557+0000","last_unstale":"2024-08-17T17:47:33.488857+0000","last_undegraded":"2024-08-17T17:47:33.488857+0000","last_fullsized":"2024-08-17T17:47:33.488857+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:33.714108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":1062,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485542+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:47:34.485542+0000","last_peered":"2024-08-17T17:47:34.485542+0000","last_clean":"2024-08-17T17:47:34.485542+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:47:34.485542+0000","last_undegraded":"2024-08-17T17:47:34.485542+0000","last_fullsized":"2024-08-17T17:47:34.485542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":786,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485265+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:47:34.485265+0000","last_peered":"2024-08-17T17:47:34.485265+0000","last_clean":"2024-08-17T17:47:34.485265+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:47:34.485265+0000","last_undegraded":"2024-08-17T17:47:34.485265+0000","last_fullsized":"2024-08-17T17:47:34.485265+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"455'11864","reported_seq":11920,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.162506+0000","last_change":"2024-08-17T17:44:36.611239+0000","last_active":"2024-08-17T17:47:36.162506+0000","last_peered":"2024-08-17T17:47:36.162506+0000","last_clean":"2024-08-17T17:47:36.162506+0000","last_became_active":"2024-08-17T17:44:36.610907+0000","last_became_peered":"2024-08-17T17:44:36.610907+0000","last_unstale":"2024-08-17T17:47:36.162506+0000","last_undegraded":"2024-08-17T17:47:36.162506+0000","last_fullsized":"2024-08-17T17:47:36.162506+0000","mapping_epoch":410,"log_start":"429'5917","ondisk_log_start":"429'5917","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:43.474878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320536,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11864,"num_write_kb":496961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":598,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497310+0000","last_change":"2024-08-17T17:47:11.533328+0000","last_active":"2024-08-17T17:47:34.497310+0000","last_peered":"2024-08-17T17:47:34.497310+0000","last_clean":"2024-08-17T17:47:34.497310+0000","last_became_active":"2024-08-17T17:47:11.532573+0000","last_became_peered":"2024-08-17T17:47:11.532573+0000","last_unstale":"2024-08-17T17:47:34.497310+0000","last_undegraded":"2024-08-17T17:47:34.497310+0000","last_fullsized":"2024-08-17T17:47:34.497310+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:42.070882+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":783,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485441+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:47:34.485441+0000","last_peered":"2024-08-17T17:47:34.485441+0000","last_clean":"2024-08-17T17:47:34.485441+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:47:34.485441+0000","last_undegraded":"2024-08-17T17:47:34.485441+0000","last_fullsized":"2024-08-17T17:47:34.485441+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1026,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491459+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:47:33.491459+0000","last_peered":"2024-08-17T17:47:33.491459+0000","last_clean":"2024-08-17T17:47:33.491459+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:47:33.491459+0000","last_undegraded":"2024-08-17T17:47:33.491459+0000","last_fullsized":"2024-08-17T17:47:33.491459+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":780,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497147+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:47:34.497147+0000","last_peered":"2024-08-17T17:47:34.497147+0000","last_clean":"2024-08-17T17:47:34.497147+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:47:34.497147+0000","last_undegraded":"2024-08-17T17:47:34.497147+0000","last_fullsized":"2024-08-17T17:47:34.497147+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"455'12023","reported_seq":12078,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.136232+0000","last_change":"2024-08-17T17:44:36.966926+0000","last_active":"2024-08-17T17:47:36.136232+0000","last_peered":"2024-08-17T17:47:36.136232+0000","last_clean":"2024-08-17T17:47:36.136232+0000","last_became_active":"2024-08-17T17:44:36.966608+0000","last_became_peered":"2024-08-17T17:44:36.966608+0000","last_unstale":"2024-08-17T17:47:36.136232+0000","last_undegraded":"2024-08-17T17:47:36.136232+0000","last_fullsized":"2024-08-17T17:47:36.136232+0000","mapping_epoch":410,"log_start":"430'6115","ondisk_log_start":"430'6115","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:39.384622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242417664,"num_objects":3699,"num_object_clones":0,"num_object_copies":11097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12023,"num_write_kb":503104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":473,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.498504+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:47:33.498504+0000","last_peered":"2024-08-17T17:47:33.498504+0000","last_clean":"2024-08-17T17:47:33.498504+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:47:33.498504+0000","last_undegraded":"2024-08-17T17:47:33.498504+0000","last_fullsized":"2024-08-17T17:47:33.498504+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":32,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489968+0000","last_change":"2024-08-17T17:47:08.512020+0000","last_active":"2024-08-17T17:47:34.489968+0000","last_peered":"2024-08-17T17:47:34.489968+0000","last_clean":"2024-08-17T17:47:34.489968+0000","last_became_active":"2024-08-17T17:47:08.497611+0000","last_became_peered":"2024-08-17T17:47:08.497611+0000","last_unstale":"2024-08-17T17:47:34.489968+0000","last_undegraded":"2024-08-17T17:47:34.489968+0000","last_fullsized":"2024-08-17T17:47:34.489968+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:50.973965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1292,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485805+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:47:34.485805+0000","last_peered":"2024-08-17T17:47:34.485805+0000","last_clean":"2024-08-17T17:47:34.485805+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:47:34.485805+0000","last_undegraded":"2024-08-17T17:47:34.485805+0000","last_fullsized":"2024-08-17T17:47:34.485805+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.488988+0000","last_change":"2024-08-17T17:47:08.497464+0000","last_active":"2024-08-17T17:47:34.488988+0000","last_peered":"2024-08-17T17:47:34.488988+0000","last_clean":"2024-08-17T17:47:34.488988+0000","last_became_active":"2024-08-17T17:47:08.496703+0000","last_became_peered":"2024-08-17T17:47:08.496703+0000","last_unstale":"2024-08-17T17:47:34.488988+0000","last_undegraded":"2024-08-17T17:47:34.488988+0000","last_fullsized":"2024-08-17T17:47:34.488988+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:04.569893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"455'12021","reported_seq":20457,"reported_epoch":456,"state":"peering","last_fresh":"2024-08-17T17:47:38.508473+0000","last_change":"2024-08-17T17:47:38.508678+0000","last_active":"2024-08-17T17:47:38.508473+0000","last_peered":"2024-08-17T17:47:38.502932+0000","last_clean":"2024-08-17T17:47:38.502932+0000","last_became_active":"2024-08-17T17:47:09.926223+0000","last_became_peered":"2024-08-17T17:47:09.926223+0000","last_unstale":"2024-08-17T17:47:38.508473+0000","last_undegraded":"2024-08-17T17:47:38.508473+0000","last_fullsized":"2024-08-17T17:47:38.508473+0000","mapping_epoch":456,"log_start":"426'4801","ondisk_log_start":"426'4801","created":410,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:44.727680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236650496,"num_objects":3611,"num_object_clones":0,"num_object_copies":10833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12021,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4179,"num_bytes_recovered":273874944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,4],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":457,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489570+0000","last_change":"2024-08-17T17:47:11.536486+0000","last_active":"2024-08-17T17:47:34.489570+0000","last_peered":"2024-08-17T17:47:34.489570+0000","last_clean":"2024-08-17T17:47:34.489570+0000","last_became_active":"2024-08-17T17:47:11.536006+0000","last_became_peered":"2024-08-17T17:47:11.536006+0000","last_unstale":"2024-08-17T17:47:34.489570+0000","last_undegraded":"2024-08-17T17:47:34.489570+0000","last_fullsized":"2024-08-17T17:47:34.489570+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:52.212639+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":792,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497375+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:47:34.497375+0000","last_peered":"2024-08-17T17:47:34.497375+0000","last_clean":"2024-08-17T17:47:34.497375+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:47:34.497375+0000","last_undegraded":"2024-08-17T17:47:34.497375+0000","last_fullsized":"2024-08-17T17:47:34.497375+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":89,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491309+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:47:33.491309+0000","last_peered":"2024-08-17T17:47:33.491309+0000","last_clean":"2024-08-17T17:47:33.491309+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:47:33.491309+0000","last_undegraded":"2024-08-17T17:47:33.491309+0000","last_fullsized":"2024-08-17T17:47:33.491309+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":401,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497347+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:47:33.497347+0000","last_peered":"2024-08-17T17:47:33.497347+0000","last_clean":"2024-08-17T17:47:33.497347+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:47:33.497347+0000","last_undegraded":"2024-08-17T17:47:33.497347+0000","last_fullsized":"2024-08-17T17:47:33.497347+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":363,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497683+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:47:33.497683+0000","last_peered":"2024-08-17T17:47:33.497683+0000","last_clean":"2024-08-17T17:47:33.497683+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:47:33.497683+0000","last_undegraded":"2024-08-17T17:47:33.497683+0000","last_fullsized":"2024-08-17T17:47:33.497683+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":373,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497770+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:47:33.497770+0000","last_peered":"2024-08-17T17:47:33.497770+0000","last_clean":"2024-08-17T17:47:33.497770+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:47:33.497770+0000","last_undegraded":"2024-08-17T17:47:33.497770+0000","last_fullsized":"2024-08-17T17:47:33.497770+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":767,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485176+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:47:34.485176+0000","last_peered":"2024-08-17T17:47:34.485176+0000","last_clean":"2024-08-17T17:47:34.485176+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:47:34.485176+0000","last_undegraded":"2024-08-17T17:47:34.485176+0000","last_fullsized":"2024-08-17T17:47:34.485176+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":488,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.491336+0000","last_change":"2024-08-17T17:47:08.506284+0000","last_active":"2024-08-17T17:47:33.491336+0000","last_peered":"2024-08-17T17:47:33.491336+0000","last_clean":"2024-08-17T17:47:33.491336+0000","last_became_active":"2024-08-17T17:47:08.495251+0000","last_became_peered":"2024-08-17T17:47:08.495251+0000","last_unstale":"2024-08-17T17:47:33.491336+0000","last_undegraded":"2024-08-17T17:47:33.491336+0000","last_fullsized":"2024-08-17T17:47:33.491336+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:36.545229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497953+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:47:33.497953+0000","last_peered":"2024-08-17T17:47:33.497953+0000","last_clean":"2024-08-17T17:47:33.497953+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:47:33.497953+0000","last_undegraded":"2024-08-17T17:47:33.497953+0000","last_fullsized":"2024-08-17T17:47:33.497953+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489300+0000","last_change":"2024-08-17T17:47:08.528976+0000","last_active":"2024-08-17T17:47:34.489300+0000","last_peered":"2024-08-17T17:47:34.489300+0000","last_clean":"2024-08-17T17:47:34.489300+0000","last_became_active":"2024-08-17T17:47:08.528667+0000","last_became_peered":"2024-08-17T17:47:08.528667+0000","last_unstale":"2024-08-17T17:47:34.489300+0000","last_undegraded":"2024-08-17T17:47:34.489300+0000","last_fullsized":"2024-08-17T17:47:34.489300+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:02.965510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":807,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497706+0000","last_change":"2024-08-17T17:47:08.492750+0000","last_active":"2024-08-17T17:47:34.497706+0000","last_peered":"2024-08-17T17:47:34.497706+0000","last_clean":"2024-08-17T17:47:34.497706+0000","last_became_active":"2024-08-17T17:47:08.492450+0000","last_became_peered":"2024-08-17T17:47:08.492450+0000","last_unstale":"2024-08-17T17:47:34.497706+0000","last_undegraded":"2024-08-17T17:47:34.497706+0000","last_fullsized":"2024-08-17T17:47:34.497706+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:51.781922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489053+0000","last_change":"2024-08-17T17:47:08.497723+0000","last_active":"2024-08-17T17:47:34.489053+0000","last_peered":"2024-08-17T17:47:34.489053+0000","last_clean":"2024-08-17T17:47:34.489053+0000","last_became_active":"2024-08-17T17:47:08.497220+0000","last_became_peered":"2024-08-17T17:47:08.497220+0000","last_unstale":"2024-08-17T17:47:34.489053+0000","last_undegraded":"2024-08-17T17:47:34.489053+0000","last_fullsized":"2024-08-17T17:47:34.489053+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:42.330633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.11","version":"455'10561","reported_seq":16063,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.674799+0000","last_change":"2024-08-17T17:47:10.543277+0000","last_active":"2024-08-17T17:47:35.674799+0000","last_peered":"2024-08-17T17:47:35.674799+0000","last_clean":"2024-08-17T17:47:35.674799+0000","last_became_active":"2024-08-17T17:47:10.526794+0000","last_became_peered":"2024-08-17T17:47:10.526794+0000","last_unstale":"2024-08-17T17:47:35.674799+0000","last_undegraded":"2024-08-17T17:47:35.674799+0000","last_fullsized":"2024-08-17T17:47:35.674799+0000","mapping_epoch":448,"log_start":"420'3200","ondisk_log_start":"420'3200","created":437,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122945536,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5867,"num_write_kb":247776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2707,"num_bytes_recovered":177405952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":251,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488822+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:47:33.488822+0000","last_peered":"2024-08-17T17:47:33.488822+0000","last_clean":"2024-08-17T17:47:33.488822+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:47:33.488822+0000","last_undegraded":"2024-08-17T17:47:33.488822+0000","last_fullsized":"2024-08-17T17:47:33.488822+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.488857+0000","last_change":"2024-08-17T17:47:08.497544+0000","last_active":"2024-08-17T17:47:34.488857+0000","last_peered":"2024-08-17T17:47:34.488857+0000","last_clean":"2024-08-17T17:47:34.488857+0000","last_became_active":"2024-08-17T17:47:08.497140+0000","last_became_peered":"2024-08-17T17:47:08.497140+0000","last_unstale":"2024-08-17T17:47:34.488857+0000","last_undegraded":"2024-08-17T17:47:34.488857+0000","last_fullsized":"2024-08-17T17:47:34.488857+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:37.427409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"455'10749","reported_seq":16498,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.111108+0000","last_change":"2024-08-17T17:47:35.823665+0000","last_active":"2024-08-17T17:47:36.111108+0000","last_peered":"2024-08-17T17:47:36.111108+0000","last_clean":"2024-08-17T17:47:36.111108+0000","last_became_active":"2024-08-17T17:47:09.497629+0000","last_became_peered":"2024-08-17T17:47:09.497629+0000","last_unstale":"2024-08-17T17:47:36.111108+0000","last_undegraded":"2024-08-17T17:47:36.111108+0000","last_fullsized":"2024-08-17T17:47:36.111108+0000","mapping_epoch":447,"log_start":"420'3200","ondisk_log_start":"420'3200","created":437,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"455'10740","last_scrub_stamp":"2024-08-17T17:47:35.823426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.823426+0000","objects_scrubbed":1920,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:32.363618+0000","scrub_duration":0.58016829199999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124649472,"num_objects":1902,"num_object_clones":0,"num_object_copies":5706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":781,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497487+0000","last_change":"2024-08-17T17:47:08.494020+0000","last_active":"2024-08-17T17:47:34.497487+0000","last_peered":"2024-08-17T17:47:34.497487+0000","last_clean":"2024-08-17T17:47:34.497487+0000","last_became_active":"2024-08-17T17:47:08.492425+0000","last_became_peered":"2024-08-17T17:47:08.492425+0000","last_unstale":"2024-08-17T17:47:34.497487+0000","last_undegraded":"2024-08-17T17:47:34.497487+0000","last_fullsized":"2024-08-17T17:47:34.497487+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:34.453167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":784,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.484995+0000","last_change":"2024-08-17T17:47:08.499047+0000","last_active":"2024-08-17T17:47:34.484995+0000","last_peered":"2024-08-17T17:47:34.484995+0000","last_clean":"2024-08-17T17:47:34.484995+0000","last_became_active":"2024-08-17T17:47:08.498700+0000","last_became_peered":"2024-08-17T17:47:08.498700+0000","last_unstale":"2024-08-17T17:47:34.484995+0000","last_undegraded":"2024-08-17T17:47:34.484995+0000","last_fullsized":"2024-08-17T17:47:34.484995+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:58.966164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.13","version":"455'10673","reported_seq":17568,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:36.410439+0000","last_change":"2024-08-17T17:47:32.197245+0000","last_active":"2024-08-17T17:47:36.410439+0000","last_peered":"2024-08-17T17:47:36.410439+0000","last_clean":"2024-08-17T17:47:36.410439+0000","last_became_active":"2024-08-17T17:47:30.671959+0000","last_became_peered":"2024-08-17T17:47:30.671959+0000","last_unstale":"2024-08-17T17:47:36.410439+0000","last_undegraded":"2024-08-17T17:47:36.410439+0000","last_fullsized":"2024-08-17T17:47:36.410439+0000","mapping_epoch":452,"log_start":"419'1300","ondisk_log_start":"419'1300","created":437,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":5,"last_scrub":"453'10574","last_scrub_stamp":"2024-08-17T17:47:32.196982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.196982+0000","objects_scrubbed":1880,"log_size":9373,"log_dups_size":0,"ondisk_log_size":9373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:03.212356+0000","scrub_duration":583,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115736576,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":114,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489642+0000","last_change":"2024-08-17T17:47:08.490394+0000","last_active":"2024-08-17T17:47:34.489642+0000","last_peered":"2024-08-17T17:47:34.489642+0000","last_clean":"2024-08-17T17:47:34.489642+0000","last_became_active":"2024-08-17T17:47:08.490056+0000","last_became_peered":"2024-08-17T17:47:08.490056+0000","last_unstale":"2024-08-17T17:47:34.489642+0000","last_undegraded":"2024-08-17T17:47:34.489642+0000","last_fullsized":"2024-08-17T17:47:34.489642+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:16.376579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":348,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497324+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:47:33.497324+0000","last_peered":"2024-08-17T17:47:33.497324+0000","last_clean":"2024-08-17T17:47:33.497324+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:47:33.497324+0000","last_undegraded":"2024-08-17T17:47:33.497324+0000","last_fullsized":"2024-08-17T17:47:33.497324+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"455'10670","reported_seq":20180,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.679486+0000","last_change":"2024-08-17T17:47:34.512043+0000","last_active":"2024-08-17T17:47:35.679486+0000","last_peered":"2024-08-17T17:47:35.679486+0000","last_clean":"2024-08-17T17:47:35.679486+0000","last_became_active":"2024-08-17T17:47:34.488995+0000","last_became_peered":"2024-08-17T17:47:34.488995+0000","last_unstale":"2024-08-17T17:47:35.679486+0000","last_undegraded":"2024-08-17T17:47:35.679486+0000","last_fullsized":"2024-08-17T17:47:35.679486+0000","mapping_epoch":454,"log_start":"420'2600","ondisk_log_start":"420'2600","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":249,"log_size":8070,"log_dups_size":0,"ondisk_log_size":8070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123502592,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5937,"num_write_kb":250272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153419776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":312,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.485690+0000","last_change":"2024-08-17T17:47:08.491671+0000","last_active":"2024-08-17T17:47:34.485690+0000","last_peered":"2024-08-17T17:47:34.485690+0000","last_clean":"2024-08-17T17:47:34.485690+0000","last_became_active":"2024-08-17T17:47:08.490847+0000","last_became_peered":"2024-08-17T17:47:08.490847+0000","last_unstale":"2024-08-17T17:47:34.485690+0000","last_undegraded":"2024-08-17T17:47:34.485690+0000","last_fullsized":"2024-08-17T17:47:34.485690+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:01:16.320104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":160,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.496946+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:47:34.496946+0000","last_peered":"2024-08-17T17:47:34.496946+0000","last_clean":"2024-08-17T17:47:34.496946+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:47:34.496946+0000","last_undegraded":"2024-08-17T17:47:34.496946+0000","last_fullsized":"2024-08-17T17:47:34.496946+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.15","version":"455'10785","reported_seq":10914,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:37.461938+0000","last_change":"2024-08-17T17:47:09.845648+0000","last_active":"2024-08-17T17:47:37.461938+0000","last_peered":"2024-08-17T17:47:37.461938+0000","last_clean":"2024-08-17T17:47:07.487679+0000","last_became_active":"2024-08-17T17:47:09.826146+0000","last_became_peered":"2024-08-17T17:47:09.826146+0000","last_unstale":"2024-08-17T17:47:37.461938+0000","last_undegraded":"2024-08-17T17:47:37.461938+0000","last_fullsized":"2024-08-17T17:47:37.461938+0000","mapping_epoch":447,"log_start":"420'2100","ondisk_log_start":"420'2100","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8685,"log_dups_size":0,"ondisk_log_size":8685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6037,"num_write_kb":250464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1790}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":366,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497610+0000","last_change":"2024-08-17T17:47:08.489958+0000","last_active":"2024-08-17T17:47:33.497610+0000","last_peered":"2024-08-17T17:47:33.497610+0000","last_clean":"2024-08-17T17:47:33.497610+0000","last_became_active":"2024-08-17T17:47:08.489547+0000","last_became_peered":"2024-08-17T17:47:08.489547+0000","last_unstale":"2024-08-17T17:47:33.497610+0000","last_undegraded":"2024-08-17T17:47:33.497610+0000","last_fullsized":"2024-08-17T17:47:33.497610+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:01.886278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":244,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488526+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:47:33.488526+0000","last_peered":"2024-08-17T17:47:33.488526+0000","last_clean":"2024-08-17T17:47:33.488526+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:47:33.488526+0000","last_undegraded":"2024-08-17T17:47:33.488526+0000","last_fullsized":"2024-08-17T17:47:33.488526+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.14","version":"455'10809","reported_seq":10935,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:37.462717+0000","last_change":"2024-08-17T17:47:09.847278+0000","last_active":"2024-08-17T17:47:37.462717+0000","last_peered":"2024-08-17T17:47:37.462717+0000","last_clean":"2024-08-17T17:47:07.521466+0000","last_became_active":"2024-08-17T17:47:09.819831+0000","last_became_peered":"2024-08-17T17:47:09.819831+0000","last_unstale":"2024-08-17T17:47:37.462717+0000","last_undegraded":"2024-08-17T17:47:37.462717+0000","last_fullsized":"2024-08-17T17:47:37.462717+0000","mapping_epoch":447,"log_start":"420'2100","ondisk_log_start":"420'2100","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8709,"log_dups_size":0,"ondisk_log_size":8709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113115136,"num_objects":1726,"num_object_clones":0,"num_object_copies":5178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1726,"num_objects_unfound":0,"num_objects_dirty":1726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6067,"num_write_kb":249376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1726}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":366,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.497247+0000","last_change":"2024-08-17T17:47:08.491957+0000","last_active":"2024-08-17T17:47:33.497247+0000","last_peered":"2024-08-17T17:47:33.497247+0000","last_clean":"2024-08-17T17:47:33.497247+0000","last_became_active":"2024-08-17T17:47:08.491740+0000","last_became_peered":"2024-08-17T17:47:08.491740+0000","last_unstale":"2024-08-17T17:47:33.497247+0000","last_undegraded":"2024-08-17T17:47:33.497247+0000","last_fullsized":"2024-08-17T17:47:33.497247+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:05.976193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":254,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.488943+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:47:33.488943+0000","last_peered":"2024-08-17T17:47:33.488943+0000","last_clean":"2024-08-17T17:47:33.488943+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:47:33.488943+0000","last_undegraded":"2024-08-17T17:47:33.488943+0000","last_fullsized":"2024-08-17T17:47:33.488943+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.17","version":"455'10608","reported_seq":10684,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:47:36.619192+0000","last_change":"2024-08-17T17:47:09.843162+0000","last_active":"2024-08-17T17:47:36.619192+0000","last_peered":"2024-08-17T17:47:36.619192+0000","last_clean":"2024-08-17T17:47:07.415398+0000","last_became_active":"2024-08-17T17:47:09.827435+0000","last_became_peered":"2024-08-17T17:47:09.827435+0000","last_unstale":"2024-08-17T17:47:36.619192+0000","last_undegraded":"2024-08-17T17:47:36.619192+0000","last_fullsized":"2024-08-17T17:47:36.619192+0000","mapping_epoch":447,"log_start":"418'1200","ondisk_log_start":"418'1200","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9408,"log_dups_size":0,"ondisk_log_size":9408,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117178368,"num_objects":1788,"num_object_clones":0,"num_object_copies":5364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1788,"num_objects_unfound":0,"num_objects_dirty":1788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5908,"num_write_kb":246272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1788}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489083+0000","last_change":"2024-08-17T17:47:08.499009+0000","last_active":"2024-08-17T17:47:34.489083+0000","last_peered":"2024-08-17T17:47:34.489083+0000","last_clean":"2024-08-17T17:47:34.489083+0000","last_became_active":"2024-08-17T17:47:08.498614+0000","last_became_peered":"2024-08-17T17:47:08.498614+0000","last_unstale":"2024-08-17T17:47:34.489083+0000","last_undegraded":"2024-08-17T17:47:34.489083+0000","last_fullsized":"2024-08-17T17:47:34.489083+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:15.246709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":28,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.489749+0000","last_change":"2024-08-17T17:47:08.496102+0000","last_active":"2024-08-17T17:47:34.489749+0000","last_peered":"2024-08-17T17:47:34.489749+0000","last_clean":"2024-08-17T17:47:34.489749+0000","last_became_active":"2024-08-17T17:47:08.495752+0000","last_became_peered":"2024-08-17T17:47:08.495752+0000","last_unstale":"2024-08-17T17:47:34.489749+0000","last_undegraded":"2024-08-17T17:47:34.489749+0000","last_fullsized":"2024-08-17T17:47:34.489749+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.648466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"455'10778","reported_seq":10843,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:35.691718+0000","last_change":"2024-08-17T17:46:52.784123+0000","last_active":"2024-08-17T17:47:35.691718+0000","last_peered":"2024-08-17T17:47:35.691718+0000","last_clean":"2024-08-17T17:47:35.691718+0000","last_became_active":"2024-08-17T17:46:52.768323+0000","last_became_peered":"2024-08-17T17:46:52.768323+0000","last_unstale":"2024-08-17T17:47:35.691718+0000","last_undegraded":"2024-08-17T17:47:35.691718+0000","last_fullsized":"2024-08-17T17:47:35.691718+0000","mapping_epoch":437,"log_start":"419'1400","ondisk_log_start":"419'1400","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9378,"log_dups_size":0,"ondisk_log_size":9378,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122912768,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":251680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":363,"reported_epoch":454,"state":"active+clean","last_fresh":"2024-08-17T17:47:33.494719+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:47:33.494719+0000","last_peered":"2024-08-17T17:47:33.494719+0000","last_clean":"2024-08-17T17:47:33.494719+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:47:33.494719+0000","last_undegraded":"2024-08-17T17:47:33.494719+0000","last_fullsized":"2024-08-17T17:47:33.494719+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":811,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-08-17T17:47:34.497682+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:47:34.497682+0000","last_peered":"2024-08-17T17:47:34.497682+0000","last_clean":"2024-08-17T17:47:34.497682+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:47:34.497682+0000","last_undegraded":"2024-08-17T17:47:34.497682+0000","last_fullsized":"2024-08-17T17:47:34.497682+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":3834806296,"num_objects":58516,"num_object_clones":0,"num_object_copies":175548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":12311,"num_objects_unfound":0,"num_objects_dirty":58516,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156495,"num_write_kb":6497185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9558,"num_bytes_recovered":626262016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12585545728,"data_stored":12585533512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":215648,"ondisk_log_size":215648,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6532,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":110,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1014065,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201809,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2433200,"kb_used_data":1437348,"kb_used_omap":1,"kb_used_meta":995838,"kb_avail":91311952,"statfs":{"total":95995035648,"available":93503438848,"internally_reserved":0,"allocated":1471844352,"data_stored":1469606077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1019738571},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922306,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2267700,"kb_used_data":1737064,"kb_used_omap":7,"kb_used_meta":530616,"kb_avail":91477452,"statfs":{"total":95995035648,"available":93672910848,"internally_reserved":0,"allocated":1778753536,"data_stored":1776518762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":543351478},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085829,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2082532,"kb_used_data":1440088,"kb_used_omap":7,"kb_used_meta":642424,"kb_avail":91662620,"statfs":{"total":95995035648,"available":93862522880,"internally_reserved":0,"allocated":1474650112,"data_stored":1472420368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":657842617},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986719,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3352304,"kb_used_data":2414692,"kb_used_omap":1,"kb_used_meta":937598,"kb_avail":90392848,"statfs":{"total":95995035648,"available":92562276352,"internally_reserved":0,"allocated":2472644608,"data_stored":2470406295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":960100806},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435610,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152780,"kb_used_data":196480,"kb_used_omap":5,"kb_used_meta":956282,"kb_avail":92592372,"statfs":{"total":95995035648,"available":94814588928,"internally_reserved":0,"allocated":201195520,"data_stored":198981172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":979232841},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599137,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2778332,"kb_used_data":1771856,"kb_used_omap":7,"kb_used_meta":1006264,"kb_avail":90966820,"statfs":{"total":95995035648,"available":93150023680,"internally_reserved":0,"allocated":1814380544,"data_stored":1812157780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":1030414509},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762663,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2072784,"kb_used_data":1188676,"kb_used_omap":7,"kb_used_meta":884088,"kb_avail":91672368,"statfs":{"total":95995035648,"available":93872504832,"internally_reserved":0,"allocated":1217204224,"data_stored":1214992334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":905306529},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926190,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3378208,"kb_used_data":2345560,"kb_used_omap":7,"kb_used_meta":1032632,"kb_avail":90366944,"statfs":{"total":95995035648,"available":92535750656,"internally_reserved":0,"allocated":2401853440,"data_stored":2399623402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":1057415772},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2384465920,"data_stored":2384461848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1087242240,"data_stored":1087242240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1789460480,"data_stored":1789460480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":210632704,"data_stored":210632704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2430078976,"data_stored":2430074904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1463091200,"data_stored":1463091200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1771376640,"data_stored":1771372568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1449197568,"data_stored":1449197568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:41.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pgp_num 26 2024-08-17T17:47:41.935 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:41.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:42.071 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:42.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:42.202 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:42.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.345 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:42.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:42.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:42.741 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:42.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.878 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:42 smithi175 ceph-mon[101434]: pgmap v1458: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11438/171957 objects misplaced (6.652%) 2024-08-17T17:47:42.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:42 smithi064 ceph-mon[104516]: pgmap v1458: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11438/171957 objects misplaced (6.652%) 2024-08-17T17:47:42.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:42 smithi064 ceph-mon[109246]: pgmap v1458: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 11438/171957 objects misplaced (6.652%) 2024-08-17T17:47:42.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:43.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:43.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:43.141 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:43.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.277 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:43.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:43.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:43.407 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:43.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.543 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:43.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:43.678 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:43.779 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:43.818 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:43.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:43 smithi175 ceph-mon[101434]: 10.11 deep-scrub starts 2024-08-17T17:47:43.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:43.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:43 smithi064 ceph-mon[109246]: 10.11 deep-scrub starts 2024-08-17T17:47:43.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:43 smithi064 ceph-mon[104516]: 10.11 deep-scrub starts 2024-08-17T17:47:43.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:44.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.102 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:44.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:44.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:44.244 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:44.346 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:44.380 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:44.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:43] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T17:47:44.481 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:44.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:44.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:44.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:44.658 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:44.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:44.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:44.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:44 smithi064 ceph-mon[104516]: pgmap v1459: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/169227 objects misplaced (6.479%) 2024-08-17T17:47:44.805 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:44 smithi064 ceph-mon[109246]: pgmap v1459: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/169227 objects misplaced (6.479%) 2024-08-17T17:47:44.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:44.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:44 smithi175 ceph-mon[101434]: pgmap v1459: 109 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 97 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/169227 objects misplaced (6.479%) 2024-08-17T17:47:44.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.938 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:44.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:45.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:45.073 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:45.073 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:45.074 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:45.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:45.176 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.213 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:45.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.477 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.569 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:47:45.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:45.620 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:45.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:45.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:47:45.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.772 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:45.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:45.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[104516]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:47:45.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1242493721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:45.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:45.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[109246]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:47:45.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1242493721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:45.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:45.873 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:45 smithi175 ceph-mon[101434]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:47:45.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:45 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1242493721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:45.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:46.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:46.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:46.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:46.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:46.185 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:46.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:46.324 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:46.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:46.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.462 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:46.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:46.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:46.589 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:46.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:46.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:46.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.867 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:46.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:46.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[104516]: pgmap v1461: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 98 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/167859 objects misplaced (6.532%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:47:46.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:46.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[104516]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:47:46.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[104516]: 10.0 deep-scrub ok 2024-08-17T17:47:46.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[109246]: pgmap v1461: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 98 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/167859 objects misplaced (6.532%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:47:46.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:46.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[109246]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:47:46.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:46 smithi064 ceph-mon[109246]: 10.0 deep-scrub ok 2024-08-17T17:47:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:46 smithi175 ceph-mon[101434]: pgmap v1461: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 6 active+remapped+backfill_wait, 98 active+clean; 3.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10965/167859 objects misplaced (6.532%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:47:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:46 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:46 smithi175 ceph-mon[101434]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:47:46.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:46 smithi175 ceph-mon[101434]: 10.0 deep-scrub ok 2024-08-17T17:47:46.969 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:46.973 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:46.996 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:46.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.007 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:47.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:47.141 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:47.242 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:47.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:47.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.421 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.523 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:47.552 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:47.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:47.825 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:47.927 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:47.969 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:48.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:48.156 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:47:48.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:47:48.189 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:47:48.189 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:48.215 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:48.222 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:47:48.222 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:48.245 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:48.256 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:47:48.257 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:48.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:47:48.301 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:48.336 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:47:48.336 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:48.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:48.368 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:47:48.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:48.380 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:48.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:47:48.397 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:48.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:48.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:47:48.516 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:48.618 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:48.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[104516]: pgmap v1463: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 99 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9569/162621 objects misplaced (5.884%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[104516]: 10.8 deep-scrub ok 2024-08-17T17:47:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[104516]: 10.18 deep-scrub ok 2024-08-17T17:47:48.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/765940729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[109246]: pgmap v1463: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 99 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9569/162621 objects misplaced (5.884%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[109246]: 10.8 deep-scrub ok 2024-08-17T17:47:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[109246]: 10.18 deep-scrub ok 2024-08-17T17:47:48.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:48 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/765940729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:48.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:48.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:48.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:48 smithi175 ceph-mon[101434]: pgmap v1463: 109 pgs: 1 active+remapped+backfilling, 4 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 99 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9569/162621 objects misplaced (5.884%); 17 MiB/s, 265 objects/s recovering 2024-08-17T17:47:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:48 smithi175 ceph-mon[101434]: 10.8 deep-scrub ok 2024-08-17T17:47:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:48 smithi175 ceph-mon[101434]: 10.18 deep-scrub ok 2024-08-17T17:47:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:48 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/765940729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:48.934 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:49.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:49.076 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:49.177 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:49.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.208 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:49.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:49.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.359 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:49.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:49.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.787 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.925 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:49.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:50.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:50.073 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:50.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:50.214 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:50.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.352 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:50.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:50.453 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.483 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:50.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:50.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:50.728 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.766 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:50.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.903 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:50 smithi175 ceph-mon[101434]: pgmap v1464: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 101 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9234/158886 objects misplaced (5.812%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:50 smithi064 ceph-mon[104516]: pgmap v1464: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 101 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9234/158886 objects misplaced (5.812%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:50.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:50 smithi064 ceph-mon[109246]: pgmap v1464: 109 pgs: 1 active+remapped+backfilling, 2 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 101 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 9234/158886 objects misplaced (5.812%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:51.006 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:51.041 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:51.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:51.186 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:51.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:51.320 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:51.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:51.463 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:51.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:51.605 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:51.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:51.847 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:51.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:51 smithi175 ceph-mon[101434]: 10.19 deep-scrub ok 2024-08-17T17:47:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:51 smithi064 ceph-mon[104516]: 10.19 deep-scrub ok 2024-08-17T17:47:51.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:51 smithi064 ceph-mon[109246]: 10.19 deep-scrub ok 2024-08-17T17:47:51.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:51.983 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:52.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:52.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:52.230 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:52.263 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:52.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:52.401 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:52.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:52.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:52.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:52.666 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:52.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:52.794 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:52.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:52.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:52.913 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:52 smithi175 ceph-mon[101434]: pgmap v1465: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9234/152862 objects misplaced (6.041%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:52.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:52 smithi064 ceph-mon[104516]: pgmap v1465: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9234/152862 objects misplaced (6.041%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:52 smithi064 ceph-mon[109246]: pgmap v1465: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 5 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9234/152862 objects misplaced (6.041%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:47:53.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:53.066 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:53.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:53.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:53.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:53.353 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:53.456 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:53.493 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:53.595 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:53.632 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:53.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:53.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:53.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:53.875 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:53.908 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:54.010 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:54.044 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:54.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.181 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:54.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:47:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:47:53] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T17:47:54.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:54.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:54.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:54.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.476 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:54.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:54.577 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:54.611 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:54 smithi064 ceph-mon[104516]: pgmap v1466: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/149439 objects misplaced (5.157%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:47:54.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:54 smithi064 ceph-mon[109246]: pgmap v1466: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/149439 objects misplaced (5.157%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:47:54.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:54.749 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:54.851 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.885 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:54.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:54.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:54 smithi175 ceph-mon[101434]: pgmap v1466: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/149439 objects misplaced (5.157%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:47:54.987 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:55.016 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:55.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:55.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:55.192 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:55.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:55.281 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:55.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:47:55.382 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.406 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:55.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:55.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:55.538 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:55.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:55.670 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:55.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.808 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:55.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:55.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:55 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:47:55.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:55 smithi175 ceph-mon[101434]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:47:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:55.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:47:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[104516]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:47:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:55.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:47:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[109246]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:47:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:47:55.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:55.944 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:56.045 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:56.083 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:56.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:56.223 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:56.325 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:56.354 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:56.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:56.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:56.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:56.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:56.642 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:56.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[104516]: pgmap v1468: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/145101 objects misplaced (5.311%); 13 MiB/s, 208 objects/s recovering 2024-08-17T17:47:56.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[104516]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:47:56.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:47:56.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[109246]: pgmap v1468: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/145101 objects misplaced (5.311%); 13 MiB/s, 208 objects/s recovering 2024-08-17T17:47:56.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[109246]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:47:56.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:56 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:47:56.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:56.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:56.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:56.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:56 smithi175 ceph-mon[101434]: pgmap v1468: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7707/145101 objects misplaced (5.311%); 13 MiB/s, 208 objects/s recovering 2024-08-17T17:47:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:56 smithi175 ceph-mon[101434]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:47:56.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:56 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:47:56.916 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:57499 9.0 e384/375 9:03f1513d:::benchmark_data_smithi064_275711_object57498:head v 384'3528, mlcod=384'3526)", 2024-08-17T17:47:57.004 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:40:30.178859+0000", 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 446.79214299699998, 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.118600971, 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.005 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.178859+0000", 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.178859+0000", 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.178860+0000", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.043e-06 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180151+0000", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001290848 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180158+0000", 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2980000000000003e-06 2024-08-17T17:47:57.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180163+0000", 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8099999999999997e-06 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180224+0000", 2024-08-17T17:47:57.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1038999999999996e-05 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180242+0000", 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8726e-05 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297421+0000", 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11717884100000001 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297460+0000", 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8365999999999998e-05 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:57502 9.6 e384/375 9:65d252bb:::benchmark_data_smithi064_275711_object57501:head v 384'3594, mlcod=384'3592)", 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:40:30.180054+0000", 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 446.79094735400002, 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11713641, 2024-08-17T17:47:57.010 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180054+0000", 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180054+0000", 2024-08-17T17:47:57.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180055+0000", 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6700000000000002e-07 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180088+0000", 2024-08-17T17:47:57.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2696e-05 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180101+0000", 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2826999999999999e-05 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180109+0000", 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0989999999999999e-06 2024-08-17T17:47:57.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180182+0000", 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3249000000000003e-05 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.180201+0000", 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8797999999999999e-05 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297144+0000", 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11694336399999999 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297191+0000", 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6409999999999998e-05 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:57504 9.2 e384/375 9:4be920d7:::benchmark_data_smithi064_275711_object57503:head v 384'3533, mlcod=384'3529)", 2024-08-17T17:47:57.015 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:40:30.181456+0000", 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 446.78954568099999, 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.123233618, 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181456+0000", 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181456+0000", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181457+0000", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.09e-07 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181667+0000", 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000210046 2024-08-17T17:47:57.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181679+0000", 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2118e-05 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.181686+0000", 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.296946+0000", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.115259453 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.296969+0000", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3586000000000001e-05 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.304650+0000", 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007680259 2024-08-17T17:47:57.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.304690+0000", 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0061000000000002e-05 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:57508 9.0 e384/375 9:0c1a6428:::benchmark_data_smithi064_275711_object57507:head v 384'3529, mlcod=384'3526)", 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:40:30.184715+0000", 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 446.78628641799997, 2024-08-17T17:47:57.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11796403799999999, 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.184715+0000", 2024-08-17T17:47:57.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.184715+0000", 2024-08-17T17:47:57.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.184716+0000", 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2669999999999999e-06 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.184771+0000", 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4877999999999999e-05 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297418+0000", 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.112646495 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297433+0000", 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5277e-05 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297487+0000", 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4193000000000001e-05 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297509+0000", 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1500000000000001e-05 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.302641+0000", 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051317460000000004 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.302679+0000", 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8682e-05 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45827.0:57513 9.2 e384/375 9:46024428:::benchmark_data_smithi064_275711_object57512:head v 384'3534, mlcod=384'3531)", 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:40:30.189123+0000", 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 446.78187825399999, 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.117720388, 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.189123+0000", 2024-08-17T17:47:57.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.189123+0000", 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.189125+0000", 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.48e-06 2024-08-17T17:47:57.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.189177+0000", 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1789000000000002e-05 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297288+0000", 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10811172099999999 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.297298+0000", 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2310000000000002e-06 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.299000+0000", 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001701988 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.299012+0000", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2368000000000001e-05 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.306697+0000", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076845619999999998 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:40:30.306844+0000", 2024-08-17T17:47:57.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014724900000000001 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:116880 10.8 e433/410 10:1e2df1db:::benchmark_data_smithi064_306376_object116879:head v 433'7240, mlcod=433'7238)", 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:46:16.274336+0000", 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 100.696665619, 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32543261600000001, 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274336+0000", 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274336+0000", 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274337+0000", 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1399999999999995e-07 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274384+0000", 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7160000000000002e-05 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274395+0000", 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1252e-05 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.274402+0000", 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6610000000000003e-06 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586231+0000", 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31182903000000001 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586259+0000", 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7790999999999999e-05 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.599660+0000", 2024-08-17T17:47:57.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013401244 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.599769+0000", 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108564 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.038 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:116881 10.7 e433/413 10:e12e7d81:::benchmark_data_smithi064_306376_object116880:head v 433'7215, mlcod=433'7211)", 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:46:16.279375+0000", 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 100.691626955, 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32053816899999998, 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279375+0000", 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279375+0000", 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279376+0000", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3120000000000001e-06 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279452+0000", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5908999999999994e-05 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586169+0000", 2024-08-17T17:47:57.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30671693100000003 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586190+0000", 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1075999999999999e-05 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.587022+0000", 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00083255999999999998 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.587042+0000", 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9483e-05 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.599864+0000", 2024-08-17T17:47:57.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01282195 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.599913+0000", 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8948e-05 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:116884 10.8 e433/410 10:11ed0cd3:::benchmark_data_smithi064_306376_object116883:head v 433'7241, mlcod=433'7238)", 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:46:16.279435+0000", 2024-08-17T17:47:57.044 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 100.691566542, 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32550191899999997, 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279435+0000", 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279435+0000", 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279436+0000", 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.16e-07 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.279464+0000", 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8204e-05 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586176+0000", 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30671193400000002 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.586190+0000", 2024-08-17T17:47:57.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3764999999999999e-05 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.589388+0000", 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031979840000000001 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.589407+0000", 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9609000000000001e-05 2024-08-17T17:47:57.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.604842+0000", 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015434583999999999 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:46:16.604937+0000", 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4922999999999999e-05 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:165421 10.13 10:cc8e582d:::benchmark_data_smithi064_306376_object40440:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e447)", 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:08.477866+0000", 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.493135203999998, 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0954876760000001, 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:47:57.050 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 165421 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477866+0000", 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477866+0000", 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477868+0000", 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3859999999999999e-06 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477873+0000", 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7670000000000003e-06 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477882+0000", 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1770000000000004e-06 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477898+0000", 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.624e-05 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477992+0000", 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4122000000000004e-05 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477999+0000", 2024-08-17T17:47:57.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.490976+0000", 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012977215 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.490986+0000", 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8949999999999993e-06 2024-08-17T17:47:57.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.573260+0000", 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0822734030000001 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.573354+0000", 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4580000000000006e-05 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.056 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:165443 10.7 10:e2c09b6f:::benchmark_data_smithi064_306376_object40462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e447)", 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:08.477917+0000", 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.493084854000003, 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4823138629999999, 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 165443 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477917+0000", 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477917+0000", 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477917+0000", 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3300000000000002e-07 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477920+0000", 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8700000000000001e-06 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477927+0000", 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2620000000000001e-06 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.477934+0000", 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6470000000000001e-06 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.478022+0000", 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7831000000000007e-05 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.478028+0000", 2024-08-17T17:47:57.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5270000000000004e-06 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.485931+0000", 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079030300000000001 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:08.485942+0000", 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0368e-05 2024-08-17T17:47:57.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.943844+0000", 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4579021599999999 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.944001+0000", 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015667400000000001 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,6", 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.944523+0000", 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052243600000000004 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.947112+0000", 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002588613 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.952091+0000", 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049795890000000004 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.952145+0000", 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4135999999999998e-05 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.959932+0000", 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077867800000000001 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.960117+0000", 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018470800000000001 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.960231+0000", 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113699 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:165459 10.7 10:e55df51a:::benchmark_data_smithi064_306376_object40478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:09.840182+0000", 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.130819668000001, 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.130161729, 2024-08-17T17:47:57.067 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 165459 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840182+0000", 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840182+0000", 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840183+0000", 2024-08-17T17:47:57.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.384e-06 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840188+0000", 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5170000000000001e-06 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840199+0000", 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1381e-05 2024-08-17T17:47:57.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.840206+0000", 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2049999999999996e-06 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.843271+0000", 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030644890000000001 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.843283+0000", 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2133000000000001e-05 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.947145+0000", 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103862149 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.947266+0000", 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120528 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,6", 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.947787+0000", 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052104799999999998 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:57.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.956490+0000", 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087031220000000006 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.956540+0000", 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9657999999999998e-05 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.956642+0000", 2024-08-17T17:47:57.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102815 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.970174+0000", 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013531836 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.970228+0000", 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3567999999999999e-05 2024-08-17T17:47:57.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.970344+0000", 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115896 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.076 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:165466 10.7 10:e33ad732:::benchmark_data_smithi064_306376_object40485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:09.855173+0000", 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.115828485000002, 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11830718, 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 165466 2024-08-17T17:47:57.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855173+0000", 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855173+0000", 2024-08-17T17:47:57.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855174+0000", 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0800000000000005e-07 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855176+0000", 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5119999999999998e-06 2024-08-17T17:47:57.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855182+0000", 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0859999999999998e-06 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855187+0000", 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5399999999999997e-06 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855245+0000", 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8128000000000003e-05 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.855257+0000", 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2123000000000001e-05 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.949715+0000", 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094458318999999999 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.949810+0000", 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4294000000000002e-05 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,6", 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.950167+0000", 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035710400000000002 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.956553+0000", 2024-08-17T17:47:57.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063863059999999996 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.956713+0000", 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015977499999999999 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.959873+0000", 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031606170000000001 2024-08-17T17:47:57.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.973173+0000", 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013299922 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.973326+0000", 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000152819 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:09.973480+0000", 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154127 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:182768 10.13 10:cb2960e7:::benchmark_data_smithi064_306376_object57787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:26.430580+0000", 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.540421394999999, 2024-08-17T17:47:57.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.113230279, 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 182768 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430580+0000", 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430580+0000", 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430581+0000", 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.178e-06 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430586+0000", 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2819999999999998e-06 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430597+0000", 2024-08-17T17:47:57.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1e-05 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430604+0000", 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5379999999999997e-06 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430663+0000", 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8436000000000001e-05 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.430830+0000", 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016781599999999999 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4,5,7", 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.431246+0000", 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041579699999999998 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.433428+0000", 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021818219999999999 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.434300+0000", 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00087160500000000001 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.435073+0000", 2024-08-17T17:47:57.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00077298999999999998 2024-08-17T17:47:57.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.435205+0000", 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013244400000000001 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.543557+0000", 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.108351499 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.543636+0000", 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9765000000000006e-05 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.543811+0000", 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017410700000000001 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:182783 10.13 10:cd7efb0b:::benchmark_data_smithi064_306376_object57802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:26.443070+0000", 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.527931533, 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.101009638, 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:47:57.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 182783 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443070+0000", 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443070+0000", 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443071+0000", 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.358e-06 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443076+0000", 2024-08-17T17:47:57.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.763e-06 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443101+0000", 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5154999999999999e-05 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443109+0000", 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0090000000000007e-06 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443175+0000", 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5893000000000001e-05 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443320+0000", 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014458799999999999 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4,5,7", 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.443727+0000", 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040669000000000002 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.446154+0000", 2024-08-17T17:47:57.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024279190000000002 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.446669+0000", 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000514088 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.446787+0000", 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011815399999999999 2024-08-17T17:47:57.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.447260+0000", 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047345800000000001 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.543864+0000", 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096604053999999995 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.543946+0000", 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1953000000000006e-05 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:26.544080+0000", 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013355600000000001 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.5.0:181395 10.19 e459/437 10:9ee0b35b:::benchmark_data_smithi064_306376_object73653:head v 459'12490, mlcod=459'12478)", 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.052762+0000", 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.9182399139999999, 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096377793000000003, 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052762+0000", 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052762+0000", 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052762+0000", 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.39e-07 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052765+0000", 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.196e-06 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052771+0000", 2024-08-17T17:47:57.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6740000000000002e-06 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.052775+0000", 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0790000000000002e-06 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.069069+0000", 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016294427 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.069090+0000", 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0975000000000002e-05 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.149064+0000", 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.07997348 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.149140+0000", 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5623000000000004e-05 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.5.0:181555 10.19 e459/437 10:9f4cb6e2:::benchmark_data_smithi064_306376_object73894:head v 459'12581, mlcod=459'12565)", 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.534429+0000", 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.4365727010000002, 2024-08-17T17:47:57.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.098371592999999993, 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534429+0000", 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534429+0000", 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534430+0000", 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1699999999999998e-07 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534434+0000", 2024-08-17T17:47:57.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7870000000000001e-06 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534443+0000", 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8570000000000001e-06 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.534449+0000", 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2859999999999996e-06 2024-08-17T17:47:57.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.558487+0000", 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024037956999999999 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.558510+0000", 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2361000000000002e-05 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.632731+0000", 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074221131999999995 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.632801+0000", 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9596000000000002e-05 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.119 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(osd.5.0:181558 10.19 e459/437 10:9f4fcdc8:::benchmark_data_smithi064_306376_object111543:head v 459'12583, mlcod=459'12565)", 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.542379+0000", 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.4286228860000003, 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094657451000000004, 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542379+0000", 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542379+0000", 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542379+0000", 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2300000000000002e-07 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542383+0000", 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4429999999999999e-06 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542388+0000", 2024-08-17T17:47:57.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9359999999999996e-06 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.542393+0000", 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4279999999999998e-06 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.568645+0000", 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026251837 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.568665+0000", 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9894e-05 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.636995+0000", 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068330061999999997 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.637036+0000", 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1527999999999999e-05 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:198767 10.19 e459/437 10:9a08d35c:::benchmark_data_smithi064_306376_object73786:head v 459'12592, mlcod=459'12565)", 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.566826+0000", 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.4041751749999998, 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.123868226, 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566826+0000", 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566826+0000", 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566827+0000", 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6199999999999996e-07 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566834+0000", 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5010000000000001e-06 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566844+0000", 2024-08-17T17:47:57.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0519999999999999e-05 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.566851+0000", 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0129999999999996e-06 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.627626+0000", 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060774160000000001 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.627646+0000", 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0462000000000002e-05 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.690647+0000", 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063001368000000002 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.690695+0000", 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7339999999999997e-05 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:198792 10.19 e459/437 10:9e7386cc:::benchmark_data_smithi064_306376_object73811:head v 459'12593, mlcod=459'12579)", 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.578188+0000", 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.3928135230000001, 2024-08-17T17:47:57.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11253497, 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578188+0000", 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578188+0000", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578188+0000", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9600000000000001e-07 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578191+0000", 2024-08-17T17:47:57.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8899999999999999e-06 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578196+0000", 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1780000000000002e-06 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.578201+0000", 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7910000000000004e-06 2024-08-17T17:47:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.632909+0000", 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.054708336000000003 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.632929+0000", 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9381999999999998e-05 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.690699+0000", 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.057770112999999998 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.690723+0000", 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4083999999999998e-05 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.55666.0:199008 10.19 e459/437 10:9824b083:::benchmark_data_smithi064_306376_object74027:head v 459'12622, mlcod=459'12614)", 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:49.795866+0000", 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.1751356739999999, 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093944389000000003, 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795866+0000", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795866+0000", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795867+0000", 2024-08-17T17:47:57.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9999999999999996e-07 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795872+0000", 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0089999999999998e-06 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795889+0000", 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7058000000000002e-05 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.795896+0000", 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1790000000000002e-06 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:47:57.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.810117+0000", 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014221411 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.810136+0000", 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8102999999999999e-05 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.889771+0000", 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079635221000000006 2024-08-17T17:47:57.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:49.889810+0000", 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9607999999999998e-05 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:47:57.142 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:47:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.183 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:57.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:57.459 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:57.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:57.606 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:57.627 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pgp_num to 26 2024-08-17T17:47:57.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:57.749 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:57.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:57.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:57.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:57.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:57 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:47:57.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:57 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:47:57.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:57 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:47:57.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:58.034 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:58.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:58.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.189 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:58.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:58.343 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:58.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:58.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:58.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:58.629 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:58.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:58.758 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:58.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:58.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:58.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:58 smithi175 ceph-mon[101434]: pgmap v1470: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 7572/137910 objects misplaced (5.491%); 5.9 MiB/s, 93 objects/s recovering 2024-08-17T17:47:58.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:58 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:47:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:58 smithi175 ceph-mon[101434]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:47:58.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:58 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:47:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[104516]: pgmap v1470: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 7572/137910 objects misplaced (5.491%); 5.9 MiB/s, 93 objects/s recovering 2024-08-17T17:47:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:47:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[104516]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:47:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:47:58.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[109246]: pgmap v1470: 109 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 103 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 7572/137910 objects misplaced (5.491%); 5.9 MiB/s, 93 objects/s recovering 2024-08-17T17:47:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3905772963' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:47:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[109246]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:47:58.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:58 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:47:58.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:59.040 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:59.142 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:59.173 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:59.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:59.304 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:59.405 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.439 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:59.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:59.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:59.568 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:59.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.711 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.813 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:59.844 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:47:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:59.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:47:59 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:47:59.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:47:59 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:47:59.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:47:59 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:47:59.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.977 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:47:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:00.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.109 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:00.369 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:00.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:00.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:00.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.620 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:00.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:00.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:00.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:00.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.869 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:00 smithi175 ceph-mon[101434]: pgmap v1472: 109 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 103 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6242/131223 objects misplaced (4.757%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:48:00.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:00 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:48:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:00 smithi175 ceph-mon[101434]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:48:00.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:00 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:48:00.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[109246]: pgmap v1472: 109 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 103 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6242/131223 objects misplaced (4.757%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:48:00.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:48:00.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[109246]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:48:00.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:48:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[104516]: pgmap v1472: 109 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 103 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6242/131223 objects misplaced (4.757%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:48:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:48:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[104516]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:48:00.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:00 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:48:00.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:00.995 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:01.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:01.121 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:01.222 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:01.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:01.350 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.375 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:01.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:01.501 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:01.626 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:01.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:01.753 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:01.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:01.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[104516]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:48:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:01.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:48:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[109246]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:48:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:01.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:01 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:48:01.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.020 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:02.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:01 smithi175 ceph-mon[101434]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:48:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:01 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:02.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:01 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:48:02.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:02.273 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:02.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:02.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:02.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:02.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:02.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:02.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:02.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:02.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:02.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:48:02.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[104516]: pgmap v1475: 109 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 5941/124389 objects misplaced (4.776%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:48:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:48:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[104516]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 462'11318 2024-08-17T17:48:02.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[104516]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[109246]: pgmap v1475: 109 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 5941/124389 objects misplaced (4.776%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[109246]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 462'11318 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[109246]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:48:02.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:02 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:48:02.972 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:02.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:03.074 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:03.109 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:03.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:02 smithi175 ceph-mon[101434]: pgmap v1475: 109 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s; 5941/124389 objects misplaced (4.776%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:48:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:02 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:48:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:02 smithi175 ceph-mon[101434]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 462'11318 2024-08-17T17:48:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:02 smithi175 ceph-mon[101434]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:48:03.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:02 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:48:03.212 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.240 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:03.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:03.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:03.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:03.440 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:03.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:03.508 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:03.610 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:03.637 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:03.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:03.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[109246]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:03.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[109246]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:48:03.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[109246]: 10.15 continuing backfill to osd.3 from (420'2100,463'11362] 10:aaea7d28:::benchmark_data_smithi064_306376_object94309:head to 463'11362 2024-08-17T17:48:03.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[109246]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:03.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:03.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[104516]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:03.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[104516]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:48:03.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[104516]: 10.15 continuing backfill to osd.3 from (420'2100,463'11362] 10:aaea7d28:::benchmark_data_smithi064_306376_object94309:head to 463'11362 2024-08-17T17:48:03.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[104516]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:03.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:03.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:03.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:03.944 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:04.037 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:03] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-08-17T17:48:04.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:04.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:03 smithi175 ceph-mon[101434]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:03 smithi175 ceph-mon[101434]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:48:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:03 smithi175 ceph-mon[101434]: 10.15 continuing backfill to osd.3 from (420'2100,463'11362] 10:aaea7d28:::benchmark_data_smithi064_306376_object94309:head to 463'11362 2024-08-17T17:48:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:03 smithi175 ceph-mon[101434]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 463'11362 2024-08-17T17:48:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:03 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:04.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:04.232 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:04.333 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.367 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:04.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:04.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:04.507 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:04.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:04.535 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:04.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.650 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.707 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:48:04.753 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:04.785 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:04.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:04.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:04 smithi064 ceph-mon[104516]: pgmap v1478: 109 pgs: 1 activating+remapped, 1 active, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6876/122766 objects misplaced (5.601%); 22 MiB/s, 347 objects/s recovering 2024-08-17T17:48:04.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:04 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:04 smithi064 ceph-mon[109246]: pgmap v1478: 109 pgs: 1 activating+remapped, 1 active, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6876/122766 objects misplaced (5.601%); 22 MiB/s, 347 objects/s recovering 2024-08-17T17:48:04.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:04 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:05.033 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:05.072 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:05.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:05.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:04 smithi175 ceph-mon[101434]: pgmap v1478: 109 pgs: 1 activating+remapped, 1 active, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 104 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6876/122766 objects misplaced (5.601%); 22 MiB/s, 347 objects/s recovering 2024-08-17T17:48:05.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:04 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:05.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:05.227 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:05.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:05.382 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:05.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.512 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:05.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:05.613 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:05.647 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:05.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:05.791 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:05.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:05.894 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:05.936 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:05.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:05.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:05.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[104516]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:48:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:05 smithi064 ceph-mon[109246]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:48:06.037 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:06.075 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:05 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1210414852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:06.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:05 smithi175 ceph-mon[101434]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:48:06.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.215 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:06.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:06.316 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:06.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:06.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.491 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:06.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:06.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:06.614 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:06.715 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:06.749 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:06.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:06.886 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:06.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:06.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:07.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:07.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:07.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:06 smithi175 ceph-mon[101434]: pgmap v1480: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6287/119850 objects misplaced (5.246%); 21 MiB/s, 341 objects/s recovering 2024-08-17T17:48:07.166 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:07.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:06 smithi064 ceph-mon[104516]: pgmap v1480: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6287/119850 objects misplaced (5.246%); 21 MiB/s, 341 objects/s recovering 2024-08-17T17:48:07.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:06 smithi064 ceph-mon[109246]: pgmap v1480: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6287/119850 objects misplaced (5.246%); 21 MiB/s, 341 objects/s recovering 2024-08-17T17:48:07.268 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:07.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:07.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:07.441 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:07.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:07.583 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:07.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:07.728 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:07.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:07.871 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:07.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:07 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:48:07.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:07 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:48:07.973 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:07.995 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:08.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.140 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:08.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:07 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:48:08.241 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:48:08.241 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:08.241 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T17:48:08.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:48:08.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.287 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.389 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:08.418 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:08.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:08.520 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.552 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:08.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:08.654 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:08.685 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:08.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:08.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:08.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:08 smithi064 ceph-mon[104516]: pgmap v1481: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5375/114174 objects misplaced (4.708%); 26 MiB/s, 411 objects/s recovering 2024-08-17T17:48:08.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:08 smithi064 ceph-mon[104516]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:48:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:08 smithi064 ceph-mon[109246]: pgmap v1481: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5375/114174 objects misplaced (4.708%); 26 MiB/s, 411 objects/s recovering 2024-08-17T17:48:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:08 smithi064 ceph-mon[109246]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:48:08.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:08.940 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.043 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.065 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:09.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:08 smithi175 ceph-mon[101434]: pgmap v1481: 109 pgs: 1 activating+remapped, 1 active, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5375/114174 objects misplaced (4.708%); 26 MiB/s, 411 objects/s recovering 2024-08-17T17:48:09.162 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:08 smithi175 ceph-mon[101434]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:48:09.166 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:09.187 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:09.289 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.311 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:09.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:09.567 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:09.670 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:09.695 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:09.797 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.819 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.821 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:09.822 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: 10.7 scrub starts 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:48:09.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3905377479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: 10.7 scrub starts 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:48:09.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3905377479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:09.921 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.944 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:09.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:10.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:10.072 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:10.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: 10.7 scrub starts 2024-08-17T17:48:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-08-17T17:48:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-08-17T17:48:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:48:10.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3905377479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:10.173 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:10.194 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:10.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:10.320 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:10.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:10.447 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:10.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:48:10.549 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.578 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:10.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.706 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:10.840 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[104516]: pgmap v1483: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4103/107193 objects misplaced (3.828%); 29 MiB/s, 456 objects/s recovering 2024-08-17T17:48:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:10.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:10.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[104516]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:48:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[109246]: pgmap v1483: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4103/107193 objects misplaced (3.828%); 29 MiB/s, 456 objects/s recovering 2024-08-17T17:48:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:10.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:10 smithi064 ceph-mon[109246]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:48:10.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.966 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:10.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:11.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.090 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:11.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:10 smithi175 ceph-mon[101434]: pgmap v1483: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 105 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4103/107193 objects misplaced (3.828%); 29 MiB/s, 456 objects/s recovering 2024-08-17T17:48:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:10 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:11.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:10 smithi175 ceph-mon[101434]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:48:11.191 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:11.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:11.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:11.212 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:11.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:11.340 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:11.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:11.470 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:11.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:11.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:11.601 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:11.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:11.725 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:11.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:11.848 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:11.950 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:11.971 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:12.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.119 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:12.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:12.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:11 smithi175 ceph-mon[101434]: 10.14 scrub starts 2024-08-17T17:48:12.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:11 smithi175 ceph-mon[101434]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:48:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:11 smithi064 ceph-mon[104516]: 10.14 scrub starts 2024-08-17T17:48:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:11 smithi064 ceph-mon[104516]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:48:12.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:11 smithi064 ceph-mon[109246]: 10.14 scrub starts 2024-08-17T17:48:12.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:11 smithi064 ceph-mon[109246]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:48:12.220 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:12.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:12.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.399 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:12.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:12.501 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:12.533 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:12.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:12.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:12.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:12.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:12.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:12.810 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:12.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:12.954 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:12.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:13.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:13.100 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:13.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:12 smithi175 ceph-mon[101434]: pgmap v1486: 109 pgs: 1 remapped+peering, 1 peering, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 3895/101796 objects misplaced (3.826%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:48:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:12 smithi175 ceph-mon[101434]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:12 smithi175 ceph-mon[101434]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[104516]: pgmap v1486: 109 pgs: 1 remapped+peering, 1 peering, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 3895/101796 objects misplaced (3.826%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:48:13.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[104516]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[104516]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[109246]: pgmap v1486: 109 pgs: 1 remapped+peering, 1 peering, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 3895/101796 objects misplaced (3.826%); 18 MiB/s, 287 objects/s recovering 2024-08-17T17:48:13.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[109246]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:12 smithi064 ceph-mon[109246]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 468'11602 2024-08-17T17:48:13.201 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:13.240 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:13.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.387 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:13.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:13.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:13.524 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:13.626 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.667 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:13.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:13.768 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:13.801 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:13.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:48:13.840 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:13.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:13.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:14.114 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.195 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:14.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.258 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:14.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:14.329 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:13] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-08-17T17:48:14.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:14.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:14.552 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:14.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:14.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:14.693 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:14.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:14.835 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:14.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:14.982 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:15.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:14 smithi064 ceph-mon[104516]: pgmap v1488: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 1067/100662 objects misplaced (1.060%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:48:15.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:14 smithi064 ceph-mon[109246]: pgmap v1488: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 1067/100662 objects misplaced (1.060%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:48:15.085 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:15.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:15.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:15.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:14 smithi175 ceph-mon[101434]: pgmap v1488: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 1067/100662 objects misplaced (1.060%); 19 MiB/s, 306 objects/s recovering 2024-08-17T17:48:15.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:15.262 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:15.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:15.364 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.395 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:15.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:15.496 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:15.529 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:15.631 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:15.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:15.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:15.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:15.919 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.959 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:15.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:16.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:16.092 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:16.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:16.229 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:16.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:16.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:16.361 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:16.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:16.490 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:16.592 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:16.615 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:16.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:16.743 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:16.845 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:16.873 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:16.975 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:17.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.010 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:17.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:17.112 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:17.161 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:17.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:16 smithi175 ceph-mon[101434]: pgmap v1489: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+remapped+backfill_wait, 106 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1067/97527 objects misplaced (1.094%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:48:17.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:16 smithi175 ceph-mon[101434]: 10.15 scrub starts 2024-08-17T17:48:17.164 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:16 smithi175 ceph-mon[101434]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:48:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[104516]: pgmap v1489: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+remapped+backfill_wait, 106 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1067/97527 objects misplaced (1.094%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:48:17.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[104516]: 10.15 scrub starts 2024-08-17T17:48:17.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[104516]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:48:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[109246]: pgmap v1489: 109 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+remapped+backfill_wait, 106 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1067/97527 objects misplaced (1.094%); 16 MiB/s, 251 objects/s recovering 2024-08-17T17:48:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[109246]: 10.15 scrub starts 2024-08-17T17:48:17.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:16 smithi064 ceph-mon[109246]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:48:17.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:17.298 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:17.400 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:17.433 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:17.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:17.576 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:17.678 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:17.709 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:17.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:17.841 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[104516]: 10.14 scrub starts 2024-08-17T17:48:17.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[104516]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[109246]: 10.14 scrub starts 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[109246]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:48:17.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:48:17.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:17.973 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.075 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:18.098 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:18.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:17 smithi175 ceph-mon[101434]: 10.14 scrub starts 2024-08-17T17:48:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:17 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:17 smithi175 ceph-mon[101434]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:48:18.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-08-17T17:48:18.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:18.225 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:18.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:18.349 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:18.450 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:18.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:18.575 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.605 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:18.732 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:18.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:18.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: pgmap v1492: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 2787/91695 objects misplaced (3.039%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:48:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: 10.4 scrub starts 2024-08-17T17:48:18.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: 10.14 scrub starts 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[104516]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: pgmap v1492: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 2787/91695 objects misplaced (3.039%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: 10.4 scrub starts 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: 10.14 scrub starts 2024-08-17T17:48:18.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:18 smithi064 ceph-mon[109246]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:48:18.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:18.980 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:48:19.071 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:48:19.082 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.103 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:19.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: pgmap v1492: 109 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 2787/91695 objects misplaced (3.039%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:48:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: 10.4 scrub starts 2024-08-17T17:48:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-08-17T17:48:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:48:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: 10.14 scrub starts 2024-08-17T17:48:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:18 smithi175 ceph-mon[101434]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:48:19.205 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:19.226 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:19.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.350 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:19.451 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.471 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:19.503 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:48:19.573 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:19.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:19.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:19.727 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:19.784 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1494,"stamp":"2024-08-17T17:48:18.498625+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1921718269,"num_objects":29279,"num_object_clones":0,"num_object_copies":87837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2116,"num_objects_unfound":0,"num_objects_dirty":29279,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":156594,"num_write_kb":5262098,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18863,"num_bytes_recovered":1235623383,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236958,"ondisk_log_size":236958,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":346,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13939964,"kb_used_data":6716072,"kb_used_omap":46,"kb_used_meta":7223505,"kb_avail":736021252,"statfs":{"total":767960285184,"available":753685762048,"internally_reserved":0,"allocated":6877257728,"data_stored":6858447945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47885,"internal_metadata":7396869363},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-280166400,"num_objects":-4275,"num_object_clones":0,"num_object_copies":-12825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1049,"num_objects_unfound":0,"num_objects_dirty":-4275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":679,"num_bytes_recovered":44433408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.076679"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.515976+0000","last_change":"2024-08-17T17:47:08.499597+0000","last_active":"2024-08-17T17:48:11.515976+0000","last_peered":"2024-08-17T17:48:11.515976+0000","last_clean":"2024-08-17T17:48:11.515976+0000","last_became_active":"2024-08-17T17:47:08.499282+0000","last_became_peered":"2024-08-17T17:47:08.499282+0000","last_unstale":"2024-08-17T17:48:11.515976+0000","last_undegraded":"2024-08-17T17:48:11.515976+0000","last_fullsized":"2024-08-17T17:48:11.515976+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:03.581238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":475,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891040+0000","last_change":"2024-08-17T17:22:21.237736+0000","last_active":"2024-08-17T17:48:11.891040+0000","last_peered":"2024-08-17T17:48:11.891040+0000","last_clean":"2024-08-17T17:48:11.891040+0000","last_became_active":"2024-08-17T17:22:21.209538+0000","last_became_peered":"2024-08-17T17:22:21.209538+0000","last_unstale":"2024-08-17T17:48:11.891040+0000","last_undegraded":"2024-08-17T17:48:11.891040+0000","last_fullsized":"2024-08-17T17:48:11.891040+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:57.815781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504417+0000","last_change":"2024-08-17T17:47:08.492972+0000","last_active":"2024-08-17T17:48:11.504417+0000","last_peered":"2024-08-17T17:48:11.504417+0000","last_clean":"2024-08-17T17:48:11.504417+0000","last_became_active":"2024-08-17T17:47:08.492623+0000","last_became_peered":"2024-08-17T17:47:08.492623+0000","last_unstale":"2024-08-17T17:48:11.504417+0000","last_undegraded":"2024-08-17T17:48:11.504417+0000","last_fullsized":"2024-08-17T17:48:11.504417+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:57.182152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":815,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794820+0000","last_change":"2024-08-17T17:47:08.483488+0000","last_active":"2024-08-17T17:48:16.794820+0000","last_peered":"2024-08-17T17:48:16.794820+0000","last_clean":"2024-08-17T17:48:16.794820+0000","last_became_active":"2024-08-17T17:47:08.483171+0000","last_became_peered":"2024-08-17T17:47:08.483171+0000","last_unstale":"2024-08-17T17:48:16.794820+0000","last_undegraded":"2024-08-17T17:48:16.794820+0000","last_fullsized":"2024-08-17T17:48:16.794820+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:19.349184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"473'13376","reported_seq":13628,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.728990+0000","last_change":"2024-08-17T17:47:47.870949+0000","last_active":"2024-08-17T17:48:17.728990+0000","last_peered":"2024-08-17T17:48:17.728990+0000","last_clean":"2024-08-17T17:48:17.728990+0000","last_became_active":"2024-08-17T17:46:52.736037+0000","last_became_peered":"2024-08-17T17:46:52.736037+0000","last_unstale":"2024-08-17T17:48:17.728990+0000","last_undegraded":"2024-08-17T17:48:17.728990+0000","last_fullsized":"2024-08-17T17:48:17.728990+0000","mapping_epoch":437,"log_start":"423'4104","ondisk_log_start":"423'4104","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"459'12596","last_scrub_stamp":"2024-08-17T17:47:47.870575+0000","last_deep_scrub":"459'12596","last_deep_scrub_stamp":"2024-08-17T17:47:47.870575+0000","last_clean_scrub_stamp":"2024-08-17T17:47:47.870575+0000","objects_scrubbed":1822,"log_size":9272,"log_dups_size":0,"ondisk_log_size":9272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":17,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:36:12.011756+0000","scrub_duration":16561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57737216,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":941,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":814,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794725+0000","last_change":"2024-08-17T17:47:08.492303+0000","last_active":"2024-08-17T17:48:16.794725+0000","last_peered":"2024-08-17T17:48:16.794725+0000","last_clean":"2024-08-17T17:48:16.794725+0000","last_became_active":"2024-08-17T17:47:08.491977+0000","last_became_peered":"2024-08-17T17:47:08.491977+0000","last_unstale":"2024-08-17T17:48:16.794725+0000","last_undegraded":"2024-08-17T17:48:16.794725+0000","last_fullsized":"2024-08-17T17:48:16.794725+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:15.963818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":461,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891040+0000","last_change":"2024-08-17T17:31:36.160158+0000","last_active":"2024-08-17T17:48:11.891040+0000","last_peered":"2024-08-17T17:48:11.891040+0000","last_clean":"2024-08-17T17:48:11.891040+0000","last_became_active":"2024-08-17T17:31:36.159865+0000","last_became_peered":"2024-08-17T17:31:36.159865+0000","last_unstale":"2024-08-17T17:48:11.891040+0000","last_undegraded":"2024-08-17T17:48:11.891040+0000","last_fullsized":"2024-08-17T17:48:11.891040+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:30.950511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.19","version":"471'13372","reported_seq":13605,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.782893+0000","last_change":"2024-08-17T17:47:50.363223+0000","last_active":"2024-08-17T17:48:15.782893+0000","last_peered":"2024-08-17T17:48:15.782893+0000","last_clean":"2024-08-17T17:48:15.782893+0000","last_became_active":"2024-08-17T17:46:52.776987+0000","last_became_peered":"2024-08-17T17:46:52.776987+0000","last_unstale":"2024-08-17T17:48:15.782893+0000","last_undegraded":"2024-08-17T17:48:15.782893+0000","last_fullsized":"2024-08-17T17:48:15.782893+0000","mapping_epoch":437,"log_start":"420'1514","ondisk_log_start":"420'1514","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"459'12722","last_scrub_stamp":"2024-08-17T17:47:50.363062+0000","last_deep_scrub":"459'12722","last_deep_scrub_stamp":"2024-08-17T17:47:50.363062+0000","last_clean_scrub_stamp":"2024-08-17T17:47:50.363062+0000","objects_scrubbed":1721,"log_size":11858,"log_dups_size":0,"ondisk_log_size":11858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":10,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:29.817656+0000","scrub_duration":9.9960606560000009,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521344,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":767,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516485+0000","last_change":"2024-08-17T17:47:08.498174+0000","last_active":"2024-08-17T17:48:11.516485+0000","last_peered":"2024-08-17T17:48:11.516485+0000","last_clean":"2024-08-17T17:48:11.516485+0000","last_became_active":"2024-08-17T17:47:08.497836+0000","last_became_peered":"2024-08-17T17:47:08.497836+0000","last_unstale":"2024-08-17T17:48:11.516485+0000","last_undegraded":"2024-08-17T17:48:11.516485+0000","last_fullsized":"2024-08-17T17:48:11.516485+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:09.116261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":408,"reported_epoch":470,"state":"active+clean","last_fresh":"2024-08-17T17:48:10.505620+0000","last_change":"2024-08-17T17:31:37.159285+0000","last_active":"2024-08-17T17:48:10.505620+0000","last_peered":"2024-08-17T17:48:10.505620+0000","last_clean":"2024-08-17T17:48:10.505620+0000","last_became_active":"2024-08-17T17:31:37.158788+0000","last_became_peered":"2024-08-17T17:31:37.158788+0000","last_unstale":"2024-08-17T17:48:10.505620+0000","last_undegraded":"2024-08-17T17:48:10.505620+0000","last_fullsized":"2024-08-17T17:48:10.505620+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:25.404174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"471'11664","reported_seq":11796,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.783325+0000","last_change":"2024-08-17T17:47:33.743471+0000","last_active":"2024-08-17T17:48:15.783325+0000","last_peered":"2024-08-17T17:48:15.783325+0000","last_clean":"2024-08-17T17:48:15.783325+0000","last_became_active":"2024-08-17T17:46:52.702751+0000","last_became_peered":"2024-08-17T17:46:52.702751+0000","last_unstale":"2024-08-17T17:48:15.783325+0000","last_undegraded":"2024-08-17T17:48:15.783325+0000","last_fullsized":"2024-08-17T17:48:15.783325+0000","mapping_epoch":410,"log_start":"419'1400","ondisk_log_start":"419'1400","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'10681","last_scrub_stamp":"2024-08-17T17:47:33.743333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:33.743333+0000","objects_scrubbed":1909,"log_size":10264,"log_dups_size":0,"ondisk_log_size":10264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:19.393314+0000","scrub_duration":0.34479844399999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60686336,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":983,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":470,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891275+0000","last_change":"2024-08-17T17:22:21.262656+0000","last_active":"2024-08-17T17:48:11.891275+0000","last_peered":"2024-08-17T17:48:11.891275+0000","last_clean":"2024-08-17T17:48:11.891275+0000","last_became_active":"2024-08-17T17:22:21.212128+0000","last_became_peered":"2024-08-17T17:22:21.212128+0000","last_unstale":"2024-08-17T17:48:11.891275+0000","last_undegraded":"2024-08-17T17:48:11.891275+0000","last_fullsized":"2024-08-17T17:48:11.891275+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:29.065304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503913+0000","last_change":"2024-08-17T17:31:37.156990+0000","last_active":"2024-08-17T17:48:11.503913+0000","last_peered":"2024-08-17T17:48:11.503913+0000","last_clean":"2024-08-17T17:48:11.503913+0000","last_became_active":"2024-08-17T17:31:37.156681+0000","last_became_peered":"2024-08-17T17:31:37.156681+0000","last_unstale":"2024-08-17T17:48:11.503913+0000","last_undegraded":"2024-08-17T17:48:11.503913+0000","last_fullsized":"2024-08-17T17:48:11.503913+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.507542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"471'11543","reported_seq":13901,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:14.514664+0000","last_change":"2024-08-17T17:48:11.081130+0000","last_active":"2024-08-17T17:48:14.514664+0000","last_peered":"2024-08-17T17:48:14.514664+0000","last_clean":"2024-08-17T17:48:14.514664+0000","last_became_active":"2024-08-17T17:48:10.510696+0000","last_became_peered":"2024-08-17T17:48:10.510696+0000","last_unstale":"2024-08-17T17:48:14.514664+0000","last_undegraded":"2024-08-17T17:48:14.514664+0000","last_fullsized":"2024-08-17T17:48:14.514664+0000","mapping_epoch":469,"log_start":"418'1200","ondisk_log_start":"418'1200","created":410,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":10343,"log_dups_size":0,"ondisk_log_size":10343,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:48:16.034539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6843,"num_write_kb":246272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1118,"num_bytes_recovered":73269248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":471,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.890973+0000","last_change":"2024-08-17T17:22:21.288146+0000","last_active":"2024-08-17T17:48:11.890973+0000","last_peered":"2024-08-17T17:48:11.890973+0000","last_clean":"2024-08-17T17:48:11.890973+0000","last_became_active":"2024-08-17T17:22:21.232650+0000","last_became_peered":"2024-08-17T17:22:21.232650+0000","last_unstale":"2024-08-17T17:48:11.890973+0000","last_undegraded":"2024-08-17T17:48:11.890973+0000","last_fullsized":"2024-08-17T17:48:11.890973+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.703850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":384,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.521864+0000","last_change":"2024-08-17T17:47:08.489953+0000","last_active":"2024-08-17T17:48:11.521864+0000","last_peered":"2024-08-17T17:48:11.521864+0000","last_clean":"2024-08-17T17:48:11.521864+0000","last_became_active":"2024-08-17T17:47:08.489588+0000","last_became_peered":"2024-08-17T17:47:08.489588+0000","last_unstale":"2024-08-17T17:48:11.521864+0000","last_undegraded":"2024-08-17T17:48:11.521864+0000","last_fullsized":"2024-08-17T17:48:11.521864+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:33.896317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"473'11603","reported_seq":13782,"reported_epoch":473,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:48:17.754627+0000","last_change":"2024-08-17T17:48:17.542498+0000","last_active":"2024-08-17T17:48:17.754627+0000","last_peered":"2024-08-17T17:48:17.754627+0000","last_clean":"2024-08-17T17:48:17.754627+0000","last_became_active":"2024-08-17T17:47:09.819565+0000","last_became_peered":"2024-08-17T17:47:09.819565+0000","last_unstale":"2024-08-17T17:48:17.754627+0000","last_undegraded":"2024-08-17T17:48:17.754627+0000","last_fullsized":"2024-08-17T17:48:17.754627+0000","mapping_epoch":447,"log_start":"420'2100","ondisk_log_start":"420'2100","created":410,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9503,"log_dups_size":0,"ondisk_log_size":9503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:48:22.533765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61079552,"num_objects":932,"num_object_clones":0,"num_object_copies":2796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6861,"num_write_kb":249376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":999,"num_bytes_recovered":65470464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["0","1","3","4"],"object_location_counts":[{"shards":"0,1,4","objects":932}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":488,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891206+0000","last_change":"2024-08-17T17:31:37.156907+0000","last_active":"2024-08-17T17:48:11.891206+0000","last_peered":"2024-08-17T17:48:11.891206+0000","last_clean":"2024-08-17T17:48:11.891206+0000","last_became_active":"2024-08-17T17:31:37.156350+0000","last_became_peered":"2024-08-17T17:31:37.156350+0000","last_unstale":"2024-08-17T17:48:11.891206+0000","last_undegraded":"2024-08-17T17:48:11.891206+0000","last_fullsized":"2024-08-17T17:48:11.891206+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:32.805947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":388,"reported_epoch":470,"state":"active+clean","last_fresh":"2024-08-17T17:48:10.505610+0000","last_change":"2024-08-17T17:31:36.155476+0000","last_active":"2024-08-17T17:48:10.505610+0000","last_peered":"2024-08-17T17:48:10.505610+0000","last_clean":"2024-08-17T17:48:10.505610+0000","last_became_active":"2024-08-17T17:31:36.155160+0000","last_became_peered":"2024-08-17T17:31:36.155160+0000","last_unstale":"2024-08-17T17:48:10.505610+0000","last_undegraded":"2024-08-17T17:48:10.505610+0000","last_fullsized":"2024-08-17T17:48:10.505610+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:59.633351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"473'11683","reported_seq":14513,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.721742+0000","last_change":"2024-08-17T17:48:16.793754+0000","last_active":"2024-08-17T17:48:17.721742+0000","last_peered":"2024-08-17T17:48:17.721742+0000","last_clean":"2024-08-17T17:48:17.721742+0000","last_became_active":"2024-08-17T17:48:02.703896+0000","last_became_peered":"2024-08-17T17:48:02.703896+0000","last_unstale":"2024-08-17T17:48:17.721742+0000","last_undegraded":"2024-08-17T17:48:17.721742+0000","last_fullsized":"2024-08-17T17:48:17.721742+0000","mapping_epoch":465,"log_start":"420'2100","ondisk_log_start":"420'2100","created":410,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9583,"log_dups_size":0,"ondisk_log_size":9583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:48:21.523895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58458112,"num_objects":892,"num_object_clones":0,"num_object_copies":2676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6935,"num_write_kb":250464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":86376448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":286,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504355+0000","last_change":"2024-08-17T17:47:08.490413+0000","last_active":"2024-08-17T17:48:11.504355+0000","last_peered":"2024-08-17T17:48:11.504355+0000","last_clean":"2024-08-17T17:48:11.504355+0000","last_became_active":"2024-08-17T17:47:08.489925+0000","last_became_peered":"2024-08-17T17:47:08.489925+0000","last_unstale":"2024-08-17T17:48:11.504355+0000","last_undegraded":"2024-08-17T17:48:11.504355+0000","last_fullsized":"2024-08-17T17:48:11.504355+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:38:39.150961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":822,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.793583+0000","last_change":"2024-08-17T17:45:27.582532+0000","last_active":"2024-08-17T17:48:16.793583+0000","last_peered":"2024-08-17T17:48:16.793583+0000","last_clean":"2024-08-17T17:48:16.793583+0000","last_became_active":"2024-08-17T17:45:27.582184+0000","last_became_peered":"2024-08-17T17:45:27.582184+0000","last_unstale":"2024-08-17T17:48:16.793583+0000","last_undegraded":"2024-08-17T17:48:16.793583+0000","last_fullsized":"2024-08-17T17:48:16.793583+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:28.064920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"474'11612","reported_seq":11811,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.941346+0000","last_change":"2024-08-17T17:47:14.318183+0000","last_active":"2024-08-17T17:48:17.941346+0000","last_peered":"2024-08-17T17:48:17.941346+0000","last_clean":"2024-08-17T17:48:17.941346+0000","last_became_active":"2024-08-17T17:46:52.725691+0000","last_became_peered":"2024-08-17T17:46:52.725691+0000","last_unstale":"2024-08-17T17:48:17.941346+0000","last_undegraded":"2024-08-17T17:48:17.941346+0000","last_fullsized":"2024-08-17T17:48:17.941346+0000","mapping_epoch":410,"log_start":"420'2600","ondisk_log_start":"420'2600","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"450'10043","last_scrub_stamp":"2024-08-17T17:47:14.317813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:14.317813+0000","objects_scrubbed":2488,"log_size":9012,"log_dups_size":0,"ondisk_log_size":9012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:20.794010+0000","scrub_duration":875,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59441152,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1581,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":864,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.787119+0000","last_change":"2024-08-17T17:47:11.537662+0000","last_active":"2024-08-17T17:48:15.787119+0000","last_peered":"2024-08-17T17:48:15.787119+0000","last_clean":"2024-08-17T17:48:15.787119+0000","last_became_active":"2024-08-17T17:47:11.536791+0000","last_became_peered":"2024-08-17T17:47:11.536791+0000","last_unstale":"2024-08-17T17:48:15.787119+0000","last_undegraded":"2024-08-17T17:48:15.787119+0000","last_fullsized":"2024-08-17T17:48:15.787119+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.544218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.544218+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:54.176207+0000","scrub_duration":0.0048673179999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":64,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503862+0000","last_change":"2024-08-17T17:47:08.509122+0000","last_active":"2024-08-17T17:48:11.503862+0000","last_peered":"2024-08-17T17:48:11.503862+0000","last_clean":"2024-08-17T17:48:11.503862+0000","last_became_active":"2024-08-17T17:47:08.497368+0000","last_became_peered":"2024-08-17T17:47:08.497368+0000","last_unstale":"2024-08-17T17:48:11.503862+0000","last_undegraded":"2024-08-17T17:48:11.503862+0000","last_fullsized":"2024-08-17T17:48:11.503862+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:35:01.638171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":408,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522376+0000","last_change":"2024-08-17T17:47:08.493553+0000","last_active":"2024-08-17T17:48:11.522376+0000","last_peered":"2024-08-17T17:48:11.522376+0000","last_clean":"2024-08-17T17:48:11.522376+0000","last_became_active":"2024-08-17T17:47:08.493193+0000","last_became_peered":"2024-08-17T17:47:08.493193+0000","last_unstale":"2024-08-17T17:48:11.522376+0000","last_undegraded":"2024-08-17T17:48:11.522376+0000","last_fullsized":"2024-08-17T17:48:11.522376+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:10.520928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"471'11689","reported_seq":15610,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.782563+0000","last_change":"2024-08-17T17:47:45.608637+0000","last_active":"2024-08-17T17:48:15.782563+0000","last_peered":"2024-08-17T17:48:15.782563+0000","last_clean":"2024-08-17T17:48:15.782563+0000","last_became_active":"2024-08-17T17:47:45.580771+0000","last_became_peered":"2024-08-17T17:47:45.580771+0000","last_unstale":"2024-08-17T17:48:15.782563+0000","last_undegraded":"2024-08-17T17:48:15.782563+0000","last_fullsized":"2024-08-17T17:48:15.782563+0000","mapping_epoch":458,"log_start":"419'1300","ondisk_log_start":"419'1300","created":410,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":10389,"log_dups_size":0,"ondisk_log_size":10389,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55541760,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6942,"num_write_kb":249280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1903,"num_bytes_recovered":124715008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":531,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503913+0000","last_change":"2024-08-17T17:47:11.534000+0000","last_active":"2024-08-17T17:48:11.503913+0000","last_peered":"2024-08-17T17:48:11.503913+0000","last_clean":"2024-08-17T17:48:11.503913+0000","last_became_active":"2024-08-17T17:47:11.533623+0000","last_became_peered":"2024-08-17T17:47:11.533623+0000","last_unstale":"2024-08-17T17:48:11.503913+0000","last_undegraded":"2024-08-17T17:48:11.503913+0000","last_fullsized":"2024-08-17T17:48:11.503913+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:28.627072+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":396,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522029+0000","last_change":"2024-08-17T17:31:36.160004+0000","last_active":"2024-08-17T17:48:11.522029+0000","last_peered":"2024-08-17T17:48:11.522029+0000","last_clean":"2024-08-17T17:48:11.522029+0000","last_became_active":"2024-08-17T17:31:36.159719+0000","last_became_peered":"2024-08-17T17:31:36.159719+0000","last_unstale":"2024-08-17T17:48:11.522029+0000","last_undegraded":"2024-08-17T17:48:11.522029+0000","last_fullsized":"2024-08-17T17:48:11.522029+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:08.592075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":366,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522264+0000","last_change":"2024-08-17T17:22:21.224285+0000","last_active":"2024-08-17T17:48:11.522264+0000","last_peered":"2024-08-17T17:48:11.522264+0000","last_clean":"2024-08-17T17:48:11.522264+0000","last_became_active":"2024-08-17T17:22:21.223956+0000","last_became_peered":"2024-08-17T17:22:21.223956+0000","last_unstale":"2024-08-17T17:48:11.522264+0000","last_undegraded":"2024-08-17T17:48:11.522264+0000","last_fullsized":"2024-08-17T17:48:11.522264+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:22.696961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"471'13429","reported_seq":13672,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:14.313991+0000","last_change":"2024-08-17T17:47:45.857911+0000","last_active":"2024-08-17T17:48:14.313991+0000","last_peered":"2024-08-17T17:48:14.313991+0000","last_clean":"2024-08-17T17:48:14.313991+0000","last_became_active":"2024-08-17T17:46:52.738619+0000","last_became_peered":"2024-08-17T17:46:52.738619+0000","last_unstale":"2024-08-17T17:48:14.313991+0000","last_undegraded":"2024-08-17T17:48:14.313991+0000","last_fullsized":"2024-08-17T17:48:14.313991+0000","mapping_epoch":410,"log_start":"420'3325","ondisk_log_start":"420'3325","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"459'12710","last_scrub_stamp":"2024-08-17T17:47:45.857752+0000","last_deep_scrub":"459'12710","last_deep_scrub_stamp":"2024-08-17T17:47:45.857752+0000","last_clean_scrub_stamp":"2024-08-17T17:47:45.857752+0000","objects_scrubbed":1788,"log_size":10104,"log_dups_size":0,"ondisk_log_size":10104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":9,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:26.195980+0000","scrub_duration":8.013654828,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65601536,"num_objects":1001,"num_object_clones":0,"num_object_copies":3003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":787,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":366,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.521755+0000","last_change":"2024-08-17T17:22:21.220716+0000","last_active":"2024-08-17T17:48:11.521755+0000","last_peered":"2024-08-17T17:48:11.521755+0000","last_clean":"2024-08-17T17:48:11.521755+0000","last_became_active":"2024-08-17T17:22:21.220254+0000","last_became_peered":"2024-08-17T17:22:21.220254+0000","last_unstale":"2024-08-17T17:48:11.521755+0000","last_undegraded":"2024-08-17T17:48:11.521755+0000","last_fullsized":"2024-08-17T17:48:11.521755+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:33.032053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":381,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522455+0000","last_change":"2024-08-17T17:31:36.161644+0000","last_active":"2024-08-17T17:48:11.522455+0000","last_peered":"2024-08-17T17:48:11.522455+0000","last_clean":"2024-08-17T17:48:11.522455+0000","last_became_active":"2024-08-17T17:31:36.160907+0000","last_became_peered":"2024-08-17T17:31:36.160907+0000","last_unstale":"2024-08-17T17:48:11.522455+0000","last_undegraded":"2024-08-17T17:48:11.522455+0000","last_fullsized":"2024-08-17T17:48:11.522455+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:15.055188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"472'11576","reported_seq":11709,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.194569+0000","last_change":"2024-08-17T17:47:34.436571+0000","last_active":"2024-08-17T17:48:16.194569+0000","last_peered":"2024-08-17T17:48:16.194569+0000","last_clean":"2024-08-17T17:48:16.194569+0000","last_became_active":"2024-08-17T17:46:52.726019+0000","last_became_peered":"2024-08-17T17:46:52.726019+0000","last_unstale":"2024-08-17T17:48:16.194569+0000","last_undegraded":"2024-08-17T17:48:16.194569+0000","last_fullsized":"2024-08-17T17:48:16.194569+0000","mapping_epoch":410,"log_start":"420'3200","ondisk_log_start":"420'3200","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'10553","last_scrub_stamp":"2024-08-17T17:47:34.436437+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:34.436437+0000","objects_scrubbed":1896,"log_size":8376,"log_dups_size":0,"ondisk_log_size":8376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:51.170531+0000","scrub_duration":0.22765941100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1023,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":489,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504052+0000","last_change":"2024-08-17T17:40:22.121869+0000","last_active":"2024-08-17T17:48:11.504052+0000","last_peered":"2024-08-17T17:48:11.504052+0000","last_clean":"2024-08-17T17:48:11.504052+0000","last_became_active":"2024-08-17T17:39:41.346787+0000","last_became_peered":"2024-08-17T17:39:41.346787+0000","last_unstale":"2024-08-17T17:48:11.504052+0000","last_undegraded":"2024-08-17T17:48:11.504052+0000","last_fullsized":"2024-08-17T17:48:11.504052+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:07.877673+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503629+0000","last_change":"2024-08-17T17:47:08.498246+0000","last_active":"2024-08-17T17:48:11.503629+0000","last_peered":"2024-08-17T17:48:11.503629+0000","last_clean":"2024-08-17T17:48:11.503629+0000","last_became_active":"2024-08-17T17:47:08.497969+0000","last_became_peered":"2024-08-17T17:47:08.497969+0000","last_unstale":"2024-08-17T17:48:11.503629+0000","last_undegraded":"2024-08-17T17:48:11.503629+0000","last_fullsized":"2024-08-17T17:48:11.503629+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:39.344214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":813,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794971+0000","last_change":"2024-08-17T17:47:08.493415+0000","last_active":"2024-08-17T17:48:16.794971+0000","last_peered":"2024-08-17T17:48:16.794971+0000","last_clean":"2024-08-17T17:48:16.794971+0000","last_became_active":"2024-08-17T17:47:08.493074+0000","last_became_peered":"2024-08-17T17:47:08.493074+0000","last_unstale":"2024-08-17T17:48:16.794971+0000","last_undegraded":"2024-08-17T17:48:16.794971+0000","last_fullsized":"2024-08-17T17:48:16.794971+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:39.768357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.b","version":"474'13844","reported_seq":13995,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.981306+0000","last_change":"2024-08-17T17:44:36.610395+0000","last_active":"2024-08-17T17:48:17.981306+0000","last_peered":"2024-08-17T17:48:17.981306+0000","last_clean":"2024-08-17T17:48:17.981306+0000","last_became_active":"2024-08-17T17:44:36.610060+0000","last_became_peered":"2024-08-17T17:44:36.610060+0000","last_unstale":"2024-08-17T17:48:17.981306+0000","last_undegraded":"2024-08-17T17:48:17.981306+0000","last_fullsized":"2024-08-17T17:48:17.981306+0000","mapping_epoch":410,"log_start":"431'6603","ondisk_log_start":"431'6603","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7241,"log_dups_size":0,"ondisk_log_size":7241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:02.611123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13844,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":814,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.890764+0000","last_change":"2024-08-17T17:39:38.495347+0000","last_active":"2024-08-17T17:48:11.890764+0000","last_peered":"2024-08-17T17:48:11.890764+0000","last_clean":"2024-08-17T17:48:11.890764+0000","last_became_active":"2024-08-17T17:39:32.484887+0000","last_became_peered":"2024-08-17T17:39:32.484887+0000","last_unstale":"2024-08-17T17:48:11.890764+0000","last_undegraded":"2024-08-17T17:48:11.890764+0000","last_fullsized":"2024-08-17T17:48:11.890764+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:39:38.495255+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:39:38.495255+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:16.244487+0000","scrub_duration":0.0013583219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1067,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504231+0000","last_change":"2024-08-17T17:31:17.588508+0000","last_active":"2024-08-17T17:48:11.504231+0000","last_peered":"2024-08-17T17:48:11.504231+0000","last_clean":"2024-08-17T17:48:11.504231+0000","last_became_active":"2024-08-17T17:31:17.587929+0000","last_became_peered":"2024-08-17T17:31:17.587929+0000","last_unstale":"2024-08-17T17:48:11.504231+0000","last_undegraded":"2024-08-17T17:48:11.504231+0000","last_fullsized":"2024-08-17T17:48:11.504231+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:24.557097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503811+0000","last_change":"2024-08-17T17:47:08.494090+0000","last_active":"2024-08-17T17:48:11.503811+0000","last_peered":"2024-08-17T17:48:11.503811+0000","last_clean":"2024-08-17T17:48:11.503811+0000","last_became_active":"2024-08-17T17:47:08.493780+0000","last_became_peered":"2024-08-17T17:47:08.493780+0000","last_unstale":"2024-08-17T17:48:11.503811+0000","last_undegraded":"2024-08-17T17:48:11.503811+0000","last_fullsized":"2024-08-17T17:48:11.503811+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:01:11.668902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":380,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522183+0000","last_change":"2024-08-17T17:31:36.163114+0000","last_active":"2024-08-17T17:48:11.522183+0000","last_peered":"2024-08-17T17:48:11.522183+0000","last_clean":"2024-08-17T17:48:11.522183+0000","last_became_active":"2024-08-17T17:31:36.162790+0000","last_became_peered":"2024-08-17T17:31:36.162790+0000","last_unstale":"2024-08-17T17:48:11.522183+0000","last_undegraded":"2024-08-17T17:48:11.522183+0000","last_fullsized":"2024-08-17T17:48:11.522183+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:13.420708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"473'13233","reported_seq":13478,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.656808+0000","last_change":"2024-08-17T17:47:47.645962+0000","last_active":"2024-08-17T17:48:17.656808+0000","last_peered":"2024-08-17T17:48:17.656808+0000","last_clean":"2024-08-17T17:48:17.656808+0000","last_became_active":"2024-08-17T17:46:52.709277+0000","last_became_peered":"2024-08-17T17:46:52.709277+0000","last_unstale":"2024-08-17T17:48:17.656808+0000","last_undegraded":"2024-08-17T17:48:17.656808+0000","last_fullsized":"2024-08-17T17:48:17.656808+0000","mapping_epoch":410,"log_start":"422'3708","ondisk_log_start":"422'3708","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"459'12493","last_scrub_stamp":"2024-08-17T17:47:47.645798+0000","last_deep_scrub":"459'12493","last_deep_scrub_stamp":"2024-08-17T17:47:47.645798+0000","last_clean_scrub_stamp":"2024-08-17T17:47:47.645798+0000","objects_scrubbed":1697,"log_size":9525,"log_dups_size":0,"ondisk_log_size":9525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":16,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:43.896475+0000","scrub_duration":15257,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":614,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.811904+0000","last_change":"2024-08-17T17:40:21.830343+0000","last_active":"2024-08-17T17:48:17.811904+0000","last_peered":"2024-08-17T17:48:17.811904+0000","last_clean":"2024-08-17T17:48:17.811904+0000","last_became_active":"2024-08-17T17:39:41.344233+0000","last_became_peered":"2024-08-17T17:39:41.344233+0000","last_unstale":"2024-08-17T17:48:17.811904+0000","last_undegraded":"2024-08-17T17:48:17.811904+0000","last_fullsized":"2024-08-17T17:48:17.811904+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.830079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.830079+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:43.638540+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":854,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.890733+0000","last_change":"2024-08-17T17:17:50.477648+0000","last_active":"2024-08-17T17:48:11.890733+0000","last_peered":"2024-08-17T17:48:11.890733+0000","last_clean":"2024-08-17T17:48:11.890733+0000","last_became_active":"2024-08-17T17:17:50.477218+0000","last_became_peered":"2024-08-17T17:17:50.477218+0000","last_unstale":"2024-08-17T17:48:11.890733+0000","last_undegraded":"2024-08-17T17:48:11.890733+0000","last_fullsized":"2024-08-17T17:48:11.890733+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:06.201248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":830,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.792405+0000","last_change":"2024-08-17T17:47:08.495040+0000","last_active":"2024-08-17T17:48:16.792405+0000","last_peered":"2024-08-17T17:48:16.792405+0000","last_clean":"2024-08-17T17:48:16.792405+0000","last_became_active":"2024-08-17T17:47:08.494475+0000","last_became_peered":"2024-08-17T17:47:08.494475+0000","last_unstale":"2024-08-17T17:48:16.792405+0000","last_undegraded":"2024-08-17T17:48:16.792405+0000","last_fullsized":"2024-08-17T17:48:16.792405+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:17.169290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":813,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.795078+0000","last_change":"2024-08-17T17:18:22.905098+0000","last_active":"2024-08-17T17:48:16.795078+0000","last_peered":"2024-08-17T17:48:16.795078+0000","last_clean":"2024-08-17T17:48:16.795078+0000","last_became_active":"2024-08-17T17:18:22.904626+0000","last_became_peered":"2024-08-17T17:18:22.904626+0000","last_unstale":"2024-08-17T17:48:16.795078+0000","last_undegraded":"2024-08-17T17:48:16.795078+0000","last_fullsized":"2024-08-17T17:48:16.795078+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:26.365546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"471'11595","reported_seq":11725,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.773896+0000","last_change":"2024-08-17T17:47:34.775376+0000","last_active":"2024-08-17T17:48:15.773896+0000","last_peered":"2024-08-17T17:48:15.773896+0000","last_clean":"2024-08-17T17:48:15.773896+0000","last_became_active":"2024-08-17T17:46:52.751923+0000","last_became_peered":"2024-08-17T17:46:52.751923+0000","last_unstale":"2024-08-17T17:48:15.773896+0000","last_undegraded":"2024-08-17T17:48:15.773896+0000","last_fullsized":"2024-08-17T17:48:15.773896+0000","mapping_epoch":410,"log_start":"419'1300","ondisk_log_start":"419'1300","created":410,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'10662","last_scrub_stamp":"2024-08-17T17:47:34.775143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:34.775143+0000","objects_scrubbed":1878,"log_size":10295,"log_dups_size":0,"ondisk_log_size":10295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:46.436810+0000","scrub_duration":0.33779308600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61800448,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":935,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":539,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.890678+0000","last_change":"2024-08-17T17:41:34.379680+0000","last_active":"2024-08-17T17:48:11.890678+0000","last_peered":"2024-08-17T17:48:11.890678+0000","last_clean":"2024-08-17T17:48:11.890678+0000","last_became_active":"2024-08-17T17:41:34.379218+0000","last_became_peered":"2024-08-17T17:41:34.379218+0000","last_unstale":"2024-08-17T17:48:11.890678+0000","last_undegraded":"2024-08-17T17:48:11.890678+0000","last_fullsized":"2024-08-17T17:48:11.890678+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:01.180030+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":82,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503792+0000","last_change":"2024-08-17T17:47:08.519932+0000","last_active":"2024-08-17T17:48:11.503792+0000","last_peered":"2024-08-17T17:48:11.503792+0000","last_clean":"2024-08-17T17:48:11.503792+0000","last_became_active":"2024-08-17T17:47:08.499007+0000","last_became_peered":"2024-08-17T17:47:08.499007+0000","last_unstale":"2024-08-17T17:48:11.503792+0000","last_undegraded":"2024-08-17T17:48:11.503792+0000","last_fullsized":"2024-08-17T17:48:11.503792+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:09.837850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":808,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.811845+0000","last_change":"2024-08-17T17:31:36.160268+0000","last_active":"2024-08-17T17:48:17.811845+0000","last_peered":"2024-08-17T17:48:17.811845+0000","last_clean":"2024-08-17T17:48:17.811845+0000","last_became_active":"2024-08-17T17:31:36.159808+0000","last_became_peered":"2024-08-17T17:31:36.159808+0000","last_unstale":"2024-08-17T17:48:17.811845+0000","last_undegraded":"2024-08-17T17:48:17.811845+0000","last_fullsized":"2024-08-17T17:48:17.811845+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:41.927087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":855,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812145+0000","last_change":"2024-08-17T17:31:17.593805+0000","last_active":"2024-08-17T17:48:17.812145+0000","last_peered":"2024-08-17T17:48:17.812145+0000","last_clean":"2024-08-17T17:48:17.812145+0000","last_became_active":"2024-08-17T17:31:17.593503+0000","last_became_peered":"2024-08-17T17:31:17.593503+0000","last_unstale":"2024-08-17T17:48:17.812145+0000","last_undegraded":"2024-08-17T17:48:17.812145+0000","last_fullsized":"2024-08-17T17:48:17.812145+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:54.360879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"471'13885","reported_seq":13956,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:14.292418+0000","last_change":"2024-08-17T17:44:36.610536+0000","last_active":"2024-08-17T17:48:14.292418+0000","last_peered":"2024-08-17T17:48:14.292418+0000","last_clean":"2024-08-17T17:48:14.292418+0000","last_became_active":"2024-08-17T17:44:36.610195+0000","last_became_peered":"2024-08-17T17:44:36.610195+0000","last_unstale":"2024-08-17T17:48:14.292418+0000","last_undegraded":"2024-08-17T17:48:14.292418+0000","last_fullsized":"2024-08-17T17:48:14.292418+0000","mapping_epoch":410,"log_start":"432'6849","ondisk_log_start":"432'6849","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:16.686185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13885,"num_write_kb":506752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":490,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504063+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:48:11.504063+0000","last_peered":"2024-08-17T17:48:11.504063+0000","last_clean":"2024-08-17T17:48:11.504063+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:48:11.504063+0000","last_undegraded":"2024-08-17T17:48:11.504063+0000","last_fullsized":"2024-08-17T17:48:11.504063+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1190,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504209+0000","last_change":"2024-08-17T17:31:37.157529+0000","last_active":"2024-08-17T17:48:11.504209+0000","last_peered":"2024-08-17T17:48:11.504209+0000","last_clean":"2024-08-17T17:48:11.504209+0000","last_became_active":"2024-08-17T17:31:37.156643+0000","last_became_peered":"2024-08-17T17:31:37.156643+0000","last_unstale":"2024-08-17T17:48:11.504209+0000","last_undegraded":"2024-08-17T17:48:11.504209+0000","last_fullsized":"2024-08-17T17:48:11.504209+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:52.599703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":631,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794301+0000","last_change":"2024-08-17T17:31:36.159212+0000","last_active":"2024-08-17T17:48:16.794301+0000","last_peered":"2024-08-17T17:48:16.794301+0000","last_clean":"2024-08-17T17:48:16.794301+0000","last_became_active":"2024-08-17T17:31:36.158876+0000","last_became_peered":"2024-08-17T17:31:36.158876+0000","last_unstale":"2024-08-17T17:48:16.794301+0000","last_undegraded":"2024-08-17T17:48:16.794301+0000","last_fullsized":"2024-08-17T17:48:16.794301+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:57.173888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":139,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504228+0000","last_change":"2024-08-17T17:47:08.490132+0000","last_active":"2024-08-17T17:48:11.504228+0000","last_peered":"2024-08-17T17:48:11.504228+0000","last_clean":"2024-08-17T17:48:11.504228+0000","last_became_active":"2024-08-17T17:47:08.489723+0000","last_became_peered":"2024-08-17T17:47:08.489723+0000","last_unstale":"2024-08-17T17:48:11.504228+0000","last_undegraded":"2024-08-17T17:48:11.504228+0000","last_fullsized":"2024-08-17T17:48:11.504228+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:49:30.591521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"472'13903","reported_seq":13975,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.109175+0000","last_change":"2024-08-17T17:44:36.607785+0000","last_active":"2024-08-17T17:48:16.109175+0000","last_peered":"2024-08-17T17:48:16.109175+0000","last_clean":"2024-08-17T17:48:16.109175+0000","last_became_active":"2024-08-17T17:44:36.607472+0000","last_became_peered":"2024-08-17T17:44:36.607472+0000","last_unstale":"2024-08-17T17:48:16.109175+0000","last_undegraded":"2024-08-17T17:48:16.109175+0000","last_fullsized":"2024-08-17T17:48:16.109175+0000","mapping_epoch":410,"log_start":"433'7151","ondisk_log_start":"433'7151","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:39.734707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118161408,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13903,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":768,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516579+0000","last_change":"2024-08-17T17:47:35.108094+0000","last_active":"2024-08-17T17:48:11.516579+0000","last_peered":"2024-08-17T17:48:11.516579+0000","last_clean":"2024-08-17T17:48:11.516579+0000","last_became_active":"2024-08-17T17:47:11.534662+0000","last_became_peered":"2024-08-17T17:47:11.534662+0000","last_unstale":"2024-08-17T17:48:11.516579+0000","last_undegraded":"2024-08-17T17:48:11.516579+0000","last_fullsized":"2024-08-17T17:48:11.516579+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:47:35.107980+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.107980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:29.724447+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504364+0000","last_change":"2024-08-17T17:47:08.492880+0000","last_active":"2024-08-17T17:48:11.504364+0000","last_peered":"2024-08-17T17:48:11.504364+0000","last_clean":"2024-08-17T17:48:11.504364+0000","last_became_active":"2024-08-17T17:47:08.492557+0000","last_became_peered":"2024-08-17T17:47:08.492557+0000","last_unstale":"2024-08-17T17:48:11.504364+0000","last_undegraded":"2024-08-17T17:48:11.504364+0000","last_fullsized":"2024-08-17T17:48:11.504364+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:33.714108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":1100,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812238+0000","last_change":"2024-08-17T17:17:50.474891+0000","last_active":"2024-08-17T17:48:17.812238+0000","last_peered":"2024-08-17T17:48:17.812238+0000","last_clean":"2024-08-17T17:48:17.812238+0000","last_became_active":"2024-08-17T17:17:50.474205+0000","last_became_peered":"2024-08-17T17:17:50.474205+0000","last_unstale":"2024-08-17T17:48:17.812238+0000","last_undegraded":"2024-08-17T17:48:17.812238+0000","last_fullsized":"2024-08-17T17:48:17.812238+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:48.428320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":824,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812523+0000","last_change":"2024-08-17T17:31:17.589441+0000","last_active":"2024-08-17T17:48:17.812523+0000","last_peered":"2024-08-17T17:48:17.812523+0000","last_clean":"2024-08-17T17:48:17.812523+0000","last_became_active":"2024-08-17T17:31:17.589090+0000","last_became_peered":"2024-08-17T17:31:17.589090+0000","last_unstale":"2024-08-17T17:48:17.812523+0000","last_undegraded":"2024-08-17T17:48:17.812523+0000","last_fullsized":"2024-08-17T17:48:17.812523+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:55.917942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"472'13710","reported_seq":13783,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.159893+0000","last_change":"2024-08-17T17:44:36.611239+0000","last_active":"2024-08-17T17:48:16.159893+0000","last_peered":"2024-08-17T17:48:16.159893+0000","last_clean":"2024-08-17T17:48:16.159893+0000","last_became_active":"2024-08-17T17:44:36.610907+0000","last_became_peered":"2024-08-17T17:44:36.610907+0000","last_unstale":"2024-08-17T17:48:16.159893+0000","last_undegraded":"2024-08-17T17:48:16.159893+0000","last_fullsized":"2024-08-17T17:48:16.159893+0000","mapping_epoch":410,"log_start":"434'7817","ondisk_log_start":"434'7817","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:43.474878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119341080,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13710,"num_write_kb":496961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":635,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794674+0000","last_change":"2024-08-17T17:47:11.533328+0000","last_active":"2024-08-17T17:48:16.794674+0000","last_peered":"2024-08-17T17:48:16.794674+0000","last_clean":"2024-08-17T17:48:16.794674+0000","last_became_active":"2024-08-17T17:47:11.532573+0000","last_became_peered":"2024-08-17T17:47:11.532573+0000","last_unstale":"2024-08-17T17:48:16.794674+0000","last_undegraded":"2024-08-17T17:48:16.794674+0000","last_fullsized":"2024-08-17T17:48:16.794674+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:42.070882+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":821,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812157+0000","last_change":"2024-08-17T17:31:17.590898+0000","last_active":"2024-08-17T17:48:17.812157+0000","last_peered":"2024-08-17T17:48:17.812157+0000","last_clean":"2024-08-17T17:48:17.812157+0000","last_became_active":"2024-08-17T17:31:17.590620+0000","last_became_peered":"2024-08-17T17:31:17.590620+0000","last_unstale":"2024-08-17T17:48:17.812157+0000","last_undegraded":"2024-08-17T17:48:17.812157+0000","last_fullsized":"2024-08-17T17:48:17.812157+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:53.580295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1043,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891200+0000","last_change":"2024-08-17T17:25:52.491047+0000","last_active":"2024-08-17T17:48:11.891200+0000","last_peered":"2024-08-17T17:48:11.891200+0000","last_clean":"2024-08-17T17:48:11.891200+0000","last_became_active":"2024-08-17T17:25:52.490476+0000","last_became_peered":"2024-08-17T17:25:52.490476+0000","last_unstale":"2024-08-17T17:48:11.891200+0000","last_undegraded":"2024-08-17T17:48:11.891200+0000","last_fullsized":"2024-08-17T17:48:11.891200+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:00.365703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":816,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.793491+0000","last_change":"2024-08-17T17:31:17.594677+0000","last_active":"2024-08-17T17:48:16.793491+0000","last_peered":"2024-08-17T17:48:16.793491+0000","last_clean":"2024-08-17T17:48:16.793491+0000","last_became_active":"2024-08-17T17:31:17.594367+0000","last_became_peered":"2024-08-17T17:31:17.594367+0000","last_unstale":"2024-08-17T17:48:16.793491+0000","last_undegraded":"2024-08-17T17:48:16.793491+0000","last_fullsized":"2024-08-17T17:48:16.793491+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:32.236691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"472'13935","reported_seq":14007,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.197934+0000","last_change":"2024-08-17T17:44:36.966926+0000","last_active":"2024-08-17T17:48:16.197934+0000","last_peered":"2024-08-17T17:48:16.197934+0000","last_clean":"2024-08-17T17:48:16.197934+0000","last_became_active":"2024-08-17T17:44:36.966608+0000","last_became_peered":"2024-08-17T17:44:36.966608+0000","last_unstale":"2024-08-17T17:48:16.197934+0000","last_undegraded":"2024-08-17T17:48:16.197934+0000","last_fullsized":"2024-08-17T17:48:16.197934+0000","mapping_epoch":410,"log_start":"434'8015","ondisk_log_start":"434'8015","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:39.384622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117112832,"num_objects":1787,"num_object_clones":0,"num_object_copies":5361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13935,"num_write_kb":503104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":491,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522108+0000","last_change":"2024-08-17T17:40:20.537873+0000","last_active":"2024-08-17T17:48:11.522108+0000","last_peered":"2024-08-17T17:48:11.522108+0000","last_clean":"2024-08-17T17:48:11.522108+0000","last_became_active":"2024-08-17T17:39:41.345255+0000","last_became_peered":"2024-08-17T17:39:41.345255+0000","last_unstale":"2024-08-17T17:48:11.522108+0000","last_undegraded":"2024-08-17T17:48:11.522108+0000","last_fullsized":"2024-08-17T17:48:11.522108+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.153900+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":64,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516479+0000","last_change":"2024-08-17T17:47:08.512020+0000","last_active":"2024-08-17T17:48:11.516479+0000","last_peered":"2024-08-17T17:48:11.516479+0000","last_clean":"2024-08-17T17:48:11.516479+0000","last_became_active":"2024-08-17T17:47:08.497611+0000","last_became_peered":"2024-08-17T17:47:08.497611+0000","last_unstale":"2024-08-17T17:48:11.516479+0000","last_undegraded":"2024-08-17T17:48:11.516479+0000","last_fullsized":"2024-08-17T17:48:11.516479+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:50.973965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":1331,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812425+0000","last_change":"2024-08-17T17:31:36.160360+0000","last_active":"2024-08-17T17:48:17.812425+0000","last_peered":"2024-08-17T17:48:17.812425+0000","last_clean":"2024-08-17T17:48:17.812425+0000","last_became_active":"2024-08-17T17:31:36.159975+0000","last_became_peered":"2024-08-17T17:31:36.159975+0000","last_unstale":"2024-08-17T17:48:17.812425+0000","last_undegraded":"2024-08-17T17:48:17.812425+0000","last_fullsized":"2024-08-17T17:48:17.812425+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:50.288774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503740+0000","last_change":"2024-08-17T17:47:08.497464+0000","last_active":"2024-08-17T17:48:11.503740+0000","last_peered":"2024-08-17T17:48:11.503740+0000","last_clean":"2024-08-17T17:48:11.503740+0000","last_became_active":"2024-08-17T17:47:08.496703+0000","last_became_peered":"2024-08-17T17:47:08.496703+0000","last_unstale":"2024-08-17T17:48:11.503740+0000","last_undegraded":"2024-08-17T17:48:11.503740+0000","last_fullsized":"2024-08-17T17:48:11.503740+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:04.569893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"471'13696","reported_seq":22155,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:14.324747+0000","last_change":"2024-08-17T17:47:39.552199+0000","last_active":"2024-08-17T17:48:14.324747+0000","last_peered":"2024-08-17T17:48:14.324747+0000","last_clean":"2024-08-17T17:48:14.324747+0000","last_became_active":"2024-08-17T17:47:39.518037+0000","last_became_peered":"2024-08-17T17:47:39.518037+0000","last_unstale":"2024-08-17T17:48:14.324747+0000","last_undegraded":"2024-08-17T17:48:14.324747+0000","last_fullsized":"2024-08-17T17:48:14.324747+0000","mapping_epoch":456,"log_start":"432'6662","ondisk_log_start":"432'6662","created":410,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:31:35.749286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126877696,"num_objects":1936,"num_object_clones":0,"num_object_copies":5808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13696,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4179,"num_bytes_recovered":273874944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":489,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516060+0000","last_change":"2024-08-17T17:47:11.536486+0000","last_active":"2024-08-17T17:48:11.516060+0000","last_peered":"2024-08-17T17:48:11.516060+0000","last_clean":"2024-08-17T17:48:11.516060+0000","last_became_active":"2024-08-17T17:47:11.536006+0000","last_became_peered":"2024-08-17T17:47:11.536006+0000","last_unstale":"2024-08-17T17:48:11.516060+0000","last_undegraded":"2024-08-17T17:48:11.516060+0000","last_fullsized":"2024-08-17T17:48:11.516060+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:52.212639+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":828,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.793901+0000","last_change":"2024-08-17T17:22:21.249266+0000","last_active":"2024-08-17T17:48:16.793901+0000","last_peered":"2024-08-17T17:48:16.793901+0000","last_clean":"2024-08-17T17:48:16.793901+0000","last_became_active":"2024-08-17T17:22:21.204990+0000","last_became_peered":"2024-08-17T17:22:21.204990+0000","last_unstale":"2024-08-17T17:48:16.793901+0000","last_undegraded":"2024-08-17T17:48:16.793901+0000","last_fullsized":"2024-08-17T17:48:16.793901+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:49.204505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":107,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.890971+0000","last_change":"2024-08-17T17:41:31.610187+0000","last_active":"2024-08-17T17:48:11.890971+0000","last_peered":"2024-08-17T17:48:11.890971+0000","last_clean":"2024-08-17T17:48:11.890971+0000","last_became_active":"2024-08-17T17:41:31.605398+0000","last_became_peered":"2024-08-17T17:41:31.605398+0000","last_unstale":"2024-08-17T17:48:11.890971+0000","last_undegraded":"2024-08-17T17:48:11.890971+0000","last_fullsized":"2024-08-17T17:48:11.890971+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:23.461514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":418,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.521943+0000","last_change":"2024-08-17T17:31:37.186991+0000","last_active":"2024-08-17T17:48:11.521943+0000","last_peered":"2024-08-17T17:48:11.521943+0000","last_clean":"2024-08-17T17:48:11.521943+0000","last_became_active":"2024-08-17T17:31:37.186226+0000","last_became_peered":"2024-08-17T17:31:37.186226+0000","last_unstale":"2024-08-17T17:48:11.521943+0000","last_undegraded":"2024-08-17T17:48:11.521943+0000","last_fullsized":"2024-08-17T17:48:11.521943+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:15.638582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":381,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522402+0000","last_change":"2024-08-17T17:31:36.164759+0000","last_active":"2024-08-17T17:48:11.522402+0000","last_peered":"2024-08-17T17:48:11.522402+0000","last_clean":"2024-08-17T17:48:11.522402+0000","last_became_active":"2024-08-17T17:31:36.164427+0000","last_became_peered":"2024-08-17T17:31:36.164427+0000","last_unstale":"2024-08-17T17:48:11.522402+0000","last_undegraded":"2024-08-17T17:48:11.522402+0000","last_fullsized":"2024-08-17T17:48:11.522402+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:35.725205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":391,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522299+0000","last_change":"2024-08-17T17:31:36.162104+0000","last_active":"2024-08-17T17:48:11.522299+0000","last_peered":"2024-08-17T17:48:11.522299+0000","last_clean":"2024-08-17T17:48:11.522299+0000","last_became_active":"2024-08-17T17:31:36.160539+0000","last_became_peered":"2024-08-17T17:31:36.160539+0000","last_unstale":"2024-08-17T17:48:11.522299+0000","last_undegraded":"2024-08-17T17:48:11.522299+0000","last_fullsized":"2024-08-17T17:48:11.522299+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:20.156256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":806,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812364+0000","last_change":"2024-08-17T17:31:36.162715+0000","last_active":"2024-08-17T17:48:17.812364+0000","last_peered":"2024-08-17T17:48:17.812364+0000","last_clean":"2024-08-17T17:48:17.812364+0000","last_became_active":"2024-08-17T17:31:36.162299+0000","last_became_peered":"2024-08-17T17:31:36.162299+0000","last_unstale":"2024-08-17T17:48:17.812364+0000","last_undegraded":"2024-08-17T17:48:17.812364+0000","last_fullsized":"2024-08-17T17:48:17.812364+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:46.602732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":505,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.891050+0000","last_change":"2024-08-17T17:47:08.506284+0000","last_active":"2024-08-17T17:48:11.891050+0000","last_peered":"2024-08-17T17:48:11.891050+0000","last_clean":"2024-08-17T17:48:11.891050+0000","last_became_active":"2024-08-17T17:47:08.495251+0000","last_became_peered":"2024-08-17T17:47:08.495251+0000","last_unstale":"2024-08-17T17:48:11.891050+0000","last_undegraded":"2024-08-17T17:48:11.891050+0000","last_fullsized":"2024-08-17T17:48:11.891050+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:36.545229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504505+0000","last_change":"2024-08-17T17:31:17.591263+0000","last_active":"2024-08-17T17:48:11.504505+0000","last_peered":"2024-08-17T17:48:11.504505+0000","last_clean":"2024-08-17T17:48:11.504505+0000","last_became_active":"2024-08-17T17:31:17.590622+0000","last_became_peered":"2024-08-17T17:31:17.590622+0000","last_unstale":"2024-08-17T17:48:11.504505+0000","last_undegraded":"2024-08-17T17:48:11.504505+0000","last_fullsized":"2024-08-17T17:48:11.504505+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:20.599491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503992+0000","last_change":"2024-08-17T17:47:08.528976+0000","last_active":"2024-08-17T17:48:11.503992+0000","last_peered":"2024-08-17T17:48:11.503992+0000","last_clean":"2024-08-17T17:48:11.503992+0000","last_became_active":"2024-08-17T17:47:08.528667+0000","last_became_peered":"2024-08-17T17:47:08.528667+0000","last_unstale":"2024-08-17T17:48:11.503992+0000","last_undegraded":"2024-08-17T17:48:11.503992+0000","last_fullsized":"2024-08-17T17:48:11.503992+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:02.965510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":843,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794465+0000","last_change":"2024-08-17T17:47:08.492750+0000","last_active":"2024-08-17T17:48:16.794465+0000","last_peered":"2024-08-17T17:48:16.794465+0000","last_clean":"2024-08-17T17:48:16.794465+0000","last_became_active":"2024-08-17T17:47:08.492450+0000","last_became_peered":"2024-08-17T17:47:08.492450+0000","last_unstale":"2024-08-17T17:48:16.794465+0000","last_undegraded":"2024-08-17T17:48:16.794465+0000","last_fullsized":"2024-08-17T17:48:16.794465+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:51.781922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503845+0000","last_change":"2024-08-17T17:47:08.497723+0000","last_active":"2024-08-17T17:48:11.503845+0000","last_peered":"2024-08-17T17:48:11.503845+0000","last_clean":"2024-08-17T17:48:11.503845+0000","last_became_active":"2024-08-17T17:47:08.497220+0000","last_became_peered":"2024-08-17T17:47:08.497220+0000","last_unstale":"2024-08-17T17:48:11.503845+0000","last_undegraded":"2024-08-17T17:48:11.503845+0000","last_fullsized":"2024-08-17T17:48:11.503845+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:42.330633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.11","version":"471'11502","reported_seq":17022,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.782797+0000","last_change":"2024-08-17T17:47:42.512959+0000","last_active":"2024-08-17T17:48:15.782797+0000","last_peered":"2024-08-17T17:48:15.782797+0000","last_clean":"2024-08-17T17:48:15.782797+0000","last_became_active":"2024-08-17T17:47:10.526794+0000","last_became_peered":"2024-08-17T17:47:10.526794+0000","last_unstale":"2024-08-17T17:48:15.782797+0000","last_undegraded":"2024-08-17T17:48:15.782797+0000","last_fullsized":"2024-08-17T17:48:15.782797+0000","mapping_epoch":448,"log_start":"420'3200","ondisk_log_start":"420'3200","created":437,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":8302,"log_dups_size":0,"ondisk_log_size":8302,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6808,"num_write_kb":247776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2707,"num_bytes_recovered":177405952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504593+0000","last_change":"2024-08-17T17:31:37.156885+0000","last_active":"2024-08-17T17:48:11.504593+0000","last_peered":"2024-08-17T17:48:11.504593+0000","last_clean":"2024-08-17T17:48:11.504593+0000","last_became_active":"2024-08-17T17:31:37.156578+0000","last_became_peered":"2024-08-17T17:31:37.156578+0000","last_unstale":"2024-08-17T17:48:11.504593+0000","last_undegraded":"2024-08-17T17:48:11.504593+0000","last_fullsized":"2024-08-17T17:48:11.504593+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:06.536993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503603+0000","last_change":"2024-08-17T17:47:08.497544+0000","last_active":"2024-08-17T17:48:11.503603+0000","last_peered":"2024-08-17T17:48:11.503603+0000","last_clean":"2024-08-17T17:48:11.503603+0000","last_became_active":"2024-08-17T17:47:08.497140+0000","last_became_peered":"2024-08-17T17:47:08.497140+0000","last_unstale":"2024-08-17T17:48:11.503603+0000","last_undegraded":"2024-08-17T17:48:11.503603+0000","last_fullsized":"2024-08-17T17:48:11.503603+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:37.427409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"472'11684","reported_seq":17450,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.121667+0000","last_change":"2024-08-17T17:47:35.823665+0000","last_active":"2024-08-17T17:48:16.121667+0000","last_peered":"2024-08-17T17:48:16.121667+0000","last_clean":"2024-08-17T17:48:16.121667+0000","last_became_active":"2024-08-17T17:47:09.497629+0000","last_became_peered":"2024-08-17T17:47:09.497629+0000","last_unstale":"2024-08-17T17:48:16.121667+0000","last_undegraded":"2024-08-17T17:48:16.121667+0000","last_fullsized":"2024-08-17T17:48:16.121667+0000","mapping_epoch":447,"log_start":"420'3200","ondisk_log_start":"420'3200","created":437,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"455'10740","last_scrub_stamp":"2024-08-17T17:47:35.823426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.823426+0000","objects_scrubbed":1920,"log_size":8484,"log_dups_size":0,"ondisk_log_size":8484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:32.363618+0000","scrub_duration":0.58016829199999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63373312,"num_objects":967,"num_object_clones":0,"num_object_copies":2901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":953,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":817,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.794917+0000","last_change":"2024-08-17T17:47:08.494020+0000","last_active":"2024-08-17T17:48:16.794917+0000","last_peered":"2024-08-17T17:48:16.794917+0000","last_clean":"2024-08-17T17:48:16.794917+0000","last_became_active":"2024-08-17T17:47:08.492425+0000","last_became_peered":"2024-08-17T17:47:08.492425+0000","last_unstale":"2024-08-17T17:48:16.794917+0000","last_undegraded":"2024-08-17T17:48:16.794917+0000","last_fullsized":"2024-08-17T17:48:16.794917+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:34.453167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":822,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.811542+0000","last_change":"2024-08-17T17:47:08.499047+0000","last_active":"2024-08-17T17:48:17.811542+0000","last_peered":"2024-08-17T17:48:17.811542+0000","last_clean":"2024-08-17T17:48:17.811542+0000","last_became_active":"2024-08-17T17:47:08.498700+0000","last_became_peered":"2024-08-17T17:47:08.498700+0000","last_unstale":"2024-08-17T17:48:17.811542+0000","last_undegraded":"2024-08-17T17:48:17.811542+0000","last_fullsized":"2024-08-17T17:48:17.811542+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:58.966164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.13","version":"474'11608","reported_seq":18546,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:18.001500+0000","last_change":"2024-08-17T17:47:32.197245+0000","last_active":"2024-08-17T17:48:18.001500+0000","last_peered":"2024-08-17T17:48:18.001500+0000","last_clean":"2024-08-17T17:48:18.001500+0000","last_became_active":"2024-08-17T17:47:30.671959+0000","last_became_peered":"2024-08-17T17:47:30.671959+0000","last_unstale":"2024-08-17T17:48:18.001500+0000","last_undegraded":"2024-08-17T17:48:18.001500+0000","last_fullsized":"2024-08-17T17:48:18.001500+0000","mapping_epoch":452,"log_start":"419'1300","ondisk_log_start":"419'1300","created":437,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":5,"last_scrub":"453'10574","last_scrub_stamp":"2024-08-17T17:47:32.196982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.196982+0000","objects_scrubbed":1880,"log_size":10308,"log_dups_size":0,"ondisk_log_size":10308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.815813+0000","scrub_duration":583,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1049,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516211+0000","last_change":"2024-08-17T17:47:08.490394+0000","last_active":"2024-08-17T17:48:11.516211+0000","last_peered":"2024-08-17T17:48:11.516211+0000","last_clean":"2024-08-17T17:48:11.516211+0000","last_became_active":"2024-08-17T17:47:08.490056+0000","last_became_peered":"2024-08-17T17:47:08.490056+0000","last_unstale":"2024-08-17T17:48:11.516211+0000","last_undegraded":"2024-08-17T17:48:11.516211+0000","last_fullsized":"2024-08-17T17:48:11.516211+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:16.376579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":365,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522205+0000","last_change":"2024-08-17T17:22:21.224000+0000","last_active":"2024-08-17T17:48:11.522205+0000","last_peered":"2024-08-17T17:48:11.522205+0000","last_clean":"2024-08-17T17:48:11.522205+0000","last_became_active":"2024-08-17T17:22:21.223540+0000","last_became_peered":"2024-08-17T17:22:21.223540+0000","last_unstale":"2024-08-17T17:48:11.522205+0000","last_undegraded":"2024-08-17T17:48:11.522205+0000","last_fullsized":"2024-08-17T17:48:11.522205+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:59.367131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"471'11621","reported_seq":21148,"reported_epoch":472,"state":"active+clean","last_fresh":"2024-08-17T17:48:15.782350+0000","last_change":"2024-08-17T17:47:34.512043+0000","last_active":"2024-08-17T17:48:15.782350+0000","last_peered":"2024-08-17T17:48:15.782350+0000","last_clean":"2024-08-17T17:48:15.782350+0000","last_became_active":"2024-08-17T17:47:34.488995+0000","last_became_peered":"2024-08-17T17:47:34.488995+0000","last_unstale":"2024-08-17T17:48:15.782350+0000","last_undegraded":"2024-08-17T17:48:15.782350+0000","last_fullsized":"2024-08-17T17:48:15.782350+0000","mapping_epoch":454,"log_start":"420'2600","ondisk_log_start":"420'2600","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":249,"log_size":9021,"log_dups_size":0,"ondisk_log_size":9021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61177856,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6888,"num_write_kb":250272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153419776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":350,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.812378+0000","last_change":"2024-08-17T17:47:08.491671+0000","last_active":"2024-08-17T17:48:17.812378+0000","last_peered":"2024-08-17T17:48:17.812378+0000","last_clean":"2024-08-17T17:48:17.812378+0000","last_became_active":"2024-08-17T17:47:08.490847+0000","last_became_peered":"2024-08-17T17:47:08.490847+0000","last_unstale":"2024-08-17T17:48:17.812378+0000","last_undegraded":"2024-08-17T17:48:17.812378+0000","last_fullsized":"2024-08-17T17:48:17.812378+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:01:16.320104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":197,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.793200+0000","last_change":"2024-08-17T17:40:08.676630+0000","last_active":"2024-08-17T17:48:16.793200+0000","last_peered":"2024-08-17T17:48:16.793200+0000","last_clean":"2024-08-17T17:48:16.793200+0000","last_became_active":"2024-08-17T17:40:08.676226+0000","last_became_peered":"2024-08-17T17:40:08.676226+0000","last_unstale":"2024-08-17T17:48:16.793200+0000","last_undegraded":"2024-08-17T17:48:16.793200+0000","last_fullsized":"2024-08-17T17:48:16.793200+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:16.578613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.15","version":"471'11609","reported_seq":15735,"reported_epoch":471,"state":"active+remapped+backfilling","last_fresh":"2024-08-17T17:48:12.764278+0000","last_change":"2024-08-17T17:48:09.603616+0000","last_active":"2024-08-17T17:48:12.764278+0000","last_peered":"2024-08-17T17:48:12.764278+0000","last_clean":"2024-08-17T17:47:07.487679+0000","last_became_active":"2024-08-17T17:48:02.824794+0000","last_became_peered":"2024-08-17T17:48:02.824794+0000","last_unstale":"2024-08-17T17:48:12.764278+0000","last_undegraded":"2024-08-17T17:48:12.764278+0000","last_fullsized":"2024-08-17T17:48:12.764278+0000","mapping_epoch":465,"log_start":"420'2100","ondisk_log_start":"420'2100","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9509,"log_dups_size":0,"ondisk_log_size":9509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63307776,"num_objects":966,"num_object_clones":0,"num_object_copies":2898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":396,"num_objects_unfound":0,"num_objects_dirty":966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6861,"num_write_kb":250464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1321,"num_bytes_recovered":86573056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":966}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522331+0000","last_change":"2024-08-17T17:47:08.489958+0000","last_active":"2024-08-17T17:48:11.522331+0000","last_peered":"2024-08-17T17:48:11.522331+0000","last_clean":"2024-08-17T17:48:11.522331+0000","last_became_active":"2024-08-17T17:47:08.489547+0000","last_became_peered":"2024-08-17T17:47:08.489547+0000","last_unstale":"2024-08-17T17:48:11.522331+0000","last_undegraded":"2024-08-17T17:48:11.522331+0000","last_fullsized":"2024-08-17T17:48:11.522331+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:01.886278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":261,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504302+0000","last_change":"2024-08-17T17:31:17.586228+0000","last_active":"2024-08-17T17:48:11.504302+0000","last_peered":"2024-08-17T17:48:11.504302+0000","last_clean":"2024-08-17T17:48:11.504302+0000","last_became_active":"2024-08-17T17:31:17.585786+0000","last_became_peered":"2024-08-17T17:31:17.585786+0000","last_unstale":"2024-08-17T17:48:11.504302+0000","last_undegraded":"2024-08-17T17:48:11.504302+0000","last_fullsized":"2024-08-17T17:48:11.504302+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:39.080154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.14","version":"474'11755","reported_seq":14266,"reported_epoch":474,"state":"active+clean","last_fresh":"2024-08-17T17:48:17.993751+0000","last_change":"2024-08-17T17:48:17.832357+0000","last_active":"2024-08-17T17:48:17.993751+0000","last_peered":"2024-08-17T17:48:17.993751+0000","last_clean":"2024-08-17T17:48:17.993751+0000","last_became_active":"2024-08-17T17:48:09.515696+0000","last_became_peered":"2024-08-17T17:48:09.515696+0000","last_unstale":"2024-08-17T17:48:17.993751+0000","last_undegraded":"2024-08-17T17:48:17.993751+0000","last_fullsized":"2024-08-17T17:48:17.993751+0000","mapping_epoch":468,"log_start":"420'2100","ondisk_log_start":"420'2100","created":437,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":9655,"log_dups_size":0,"ondisk_log_size":9655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:48:22.813665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51118080,"num_objects":780,"num_object_clones":0,"num_object_copies":2340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7013,"num_write_kb":249376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1162,"num_bytes_recovered":76087296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":383,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.522107+0000","last_change":"2024-08-17T17:47:08.491957+0000","last_active":"2024-08-17T17:48:11.522107+0000","last_peered":"2024-08-17T17:48:11.522107+0000","last_clean":"2024-08-17T17:48:11.522107+0000","last_became_active":"2024-08-17T17:47:08.491740+0000","last_became_peered":"2024-08-17T17:47:08.491740+0000","last_unstale":"2024-08-17T17:48:11.522107+0000","last_undegraded":"2024-08-17T17:48:11.522107+0000","last_fullsized":"2024-08-17T17:48:11.522107+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:05.976193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":271,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.504427+0000","last_change":"2024-08-17T17:31:37.157756+0000","last_active":"2024-08-17T17:48:11.504427+0000","last_peered":"2024-08-17T17:48:11.504427+0000","last_clean":"2024-08-17T17:48:11.504427+0000","last_became_active":"2024-08-17T17:31:37.157298+0000","last_became_peered":"2024-08-17T17:31:37.157298+0000","last_unstale":"2024-08-17T17:48:11.504427+0000","last_undegraded":"2024-08-17T17:48:11.504427+0000","last_fullsized":"2024-08-17T17:48:11.504427+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:54.366611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.17","version":"471'11465","reported_seq":14706,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:14.507160+0000","last_change":"2024-08-17T17:48:10.510120+0000","last_active":"2024-08-17T17:48:14.507160+0000","last_peered":"2024-08-17T17:48:14.507160+0000","last_clean":"2024-08-17T17:48:14.507160+0000","last_became_active":"2024-08-17T17:47:55.613452+0000","last_became_peered":"2024-08-17T17:47:55.613452+0000","last_unstale":"2024-08-17T17:48:14.507160+0000","last_undegraded":"2024-08-17T17:48:14.507160+0000","last_fullsized":"2024-08-17T17:48:14.507160+0000","mapping_epoch":460,"log_start":"418'1200","ondisk_log_start":"418'1200","created":437,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:44:35.588248+0000","objects_scrubbed":0,"log_size":10265,"log_dups_size":0,"ondisk_log_size":10265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:48:15.058028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6765,"num_write_kb":246272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1549,"num_bytes_recovered":101515264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.503728+0000","last_change":"2024-08-17T17:47:08.499009+0000","last_active":"2024-08-17T17:48:11.503728+0000","last_peered":"2024-08-17T17:48:11.503728+0000","last_clean":"2024-08-17T17:48:11.503728+0000","last_became_active":"2024-08-17T17:47:08.498614+0000","last_became_peered":"2024-08-17T17:47:08.498614+0000","last_unstale":"2024-08-17T17:48:11.503728+0000","last_undegraded":"2024-08-17T17:48:11.503728+0000","last_fullsized":"2024-08-17T17:48:11.503728+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:15.246709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":60,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.516687+0000","last_change":"2024-08-17T17:47:08.496102+0000","last_active":"2024-08-17T17:48:11.516687+0000","last_peered":"2024-08-17T17:48:11.516687+0000","last_clean":"2024-08-17T17:48:11.516687+0000","last_became_active":"2024-08-17T17:47:08.495752+0000","last_became_peered":"2024-08-17T17:47:08.495752+0000","last_unstale":"2024-08-17T17:48:11.516687+0000","last_undegraded":"2024-08-17T17:48:11.516687+0000","last_fullsized":"2024-08-17T17:48:11.516687+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.648466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"471'11742","reported_seq":12234,"reported_epoch":472,"state":"active+remapped+backfilling","last_fresh":"2024-08-17T17:48:15.787339+0000","last_change":"2024-08-17T17:48:14.770556+0000","last_active":"2024-08-17T17:48:15.787339+0000","last_peered":"2024-08-17T17:48:15.787339+0000","last_clean":"2024-08-17T17:48:08.908301+0000","last_became_active":"2024-08-17T17:48:11.950064+0000","last_became_peered":"2024-08-17T17:48:11.950064+0000","last_unstale":"2024-08-17T17:48:15.787339+0000","last_undegraded":"2024-08-17T17:48:15.787339+0000","last_fullsized":"2024-08-17T17:48:15.787339+0000","mapping_epoch":470,"log_start":"419'1400","ondisk_log_start":"419'1400","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"455'10827","last_scrub_stamp":"2024-08-17T17:47:37.930480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:44:35.588248+0000","last_clean_scrub_stamp":"2024-08-17T17:47:37.930480+0000","objects_scrubbed":1890,"log_size":10342,"log_dups_size":0,"ondisk_log_size":10342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:47.592743+0000","scrub_duration":0.50466271399999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1720,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":915,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7667712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,0,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":975}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":380,"reported_epoch":471,"state":"active+clean","last_fresh":"2024-08-17T17:48:11.521706+0000","last_change":"2024-08-17T17:31:36.160325+0000","last_active":"2024-08-17T17:48:11.521706+0000","last_peered":"2024-08-17T17:48:11.521706+0000","last_clean":"2024-08-17T17:48:11.521706+0000","last_became_active":"2024-08-17T17:31:36.159627+0000","last_became_peered":"2024-08-17T17:31:36.159627+0000","last_unstale":"2024-08-17T17:48:11.521706+0000","last_undegraded":"2024-08-17T17:48:11.521706+0000","last_fullsized":"2024-08-17T17:48:11.521706+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:14.990187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":847,"reported_epoch":473,"state":"active+clean","last_fresh":"2024-08-17T17:48:16.793855+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:48:16.793855+0000","last_peered":"2024-08-17T17:48:16.793855+0000","last_clean":"2024-08-17T17:48:16.793855+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:48:16.793855+0000","last_undegraded":"2024-08-17T17:48:16.793855+0000","last_fullsized":"2024-08-17T17:48:16.793855+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":1916928024,"num_objects":29251,"num_object_clones":0,"num_object_copies":87753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2116,"num_objects_unfound":0,"num_objects_dirty":29251,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155686,"num_write_kb":5249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18714,"num_bytes_recovered":1226375168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6725382144,"data_stored":6725369928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":234286,"ondisk_log_size":234286,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6532,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29892,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":110,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5249,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201817,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1701608,"kb_used_data":713952,"kb_used_omap":1,"kb_used_meta":987646,"kb_avail":92043544,"statfs":{"total":95995035648,"available":94252589056,"internally_reserved":0,"allocated":731086848,"data_stored":728728199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1011349963},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922313,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1480220,"kb_used_data":945428,"kb_used_omap":7,"kb_used_meta":534712,"kb_avail":92264932,"statfs":{"total":95995035648,"available":94479290368,"internally_reserved":0,"allocated":968118272,"data_stored":965770901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":547545782},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085837,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1454236,"kb_used_data":771540,"kb_used_omap":7,"kb_used_meta":682616,"kb_avail":92290916,"statfs":{"total":95995035648,"available":94505897984,"internally_reserved":0,"allocated":790056960,"data_stored":787706842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":698999225},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986727,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2132536,"kb_used_data":1166956,"kb_used_omap":1,"kb_used_meta":965502,"kb_avail":91612616,"statfs":{"total":95995035648,"available":93811318784,"internally_reserved":0,"allocated":1194962944,"data_stored":1192600411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":988674502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435618,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1477268,"kb_used_data":457100,"kb_used_omap":5,"kb_used_meta":1020154,"kb_avail":92267884,"statfs":{"total":95995035648,"available":94482313216,"internally_reserved":0,"allocated":468070400,"data_stored":465727913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":1044637769},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599145,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963044,"kb_used_data":916504,"kb_used_omap":7,"kb_used_meta":1046520,"kb_avail":91782108,"statfs":{"total":95995035648,"available":93984878592,"internally_reserved":0,"allocated":938500096,"data_stored":936153112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":1071636653},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762672,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1534516,"kb_used_data":618468,"kb_used_omap":7,"kb_used_meta":916024,"kb_avail":92210636,"statfs":{"total":95995035648,"available":94423691264,"internally_reserved":0,"allocated":633311232,"data_stored":630967716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":938008993},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926198,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2196536,"kb_used_data":1126124,"kb_used_omap":7,"kb_used_meta":1070328,"kb_avail":91548616,"statfs":{"total":95995035648,"available":93745782784,"internally_reserved":0,"allocated":1153150976,"data_stored":1150792851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":1096016476},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1130827776,"data_stored":1130823704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":597819392,"data_stored":597819392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":923140096,"data_stored":923140096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":456392704,"data_stored":456392704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1181945856,"data_stored":1181941784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":768540672,"data_stored":768540672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":955191296,"data_stored":955187224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":711524352,"data_stored":711524352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:48:19.789 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:48:19.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 36 2024-08-17T17:48:19.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:19.857 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:19.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:19.989 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:19.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: 10.7 scrub starts 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1778692020' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:48:20.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[109246]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: 10.7 scrub starts 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1778692020' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:48:20.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:19 smithi064 ceph-mon[104516]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:48:20.091 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:20.112 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:20.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: 10.7 scrub starts 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1778692020' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2024-08-17T17:48:20.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:19 smithi175 ceph-mon[101434]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:48:20.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:20.237 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:20.338 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:20.359 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:20.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:20.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:20.552 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:20.552 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:20.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:20.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:20.719 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:20.744 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:20.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:20.846 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:20.849 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:20.868 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:20.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:20.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:48:20.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: pgmap v1495: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 2116/86133 objects misplaced (2.457%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:48:20.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 473'11540 2024-08-17T17:48:20.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: 10.5 scrub starts 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: 10.5 scrub ok 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: 10.7 scrub starts 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: 10.7 scrub ok 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[104516]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: pgmap v1495: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 2116/86133 objects misplaced (2.457%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 473'11540 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: 10.5 scrub starts 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: 10.5 scrub ok 2024-08-17T17:48:20.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: 10.7 scrub starts 2024-08-17T17:48:20.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: 10.7 scrub ok 2024-08-17T17:48:20.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:20 smithi064 ceph-mon[109246]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:48:20.998 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:21.100 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.121 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:21.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: pgmap v1495: 109 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 2116/86133 objects misplaced (2.457%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 473'11540 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: 10.5 scrub starts 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: 10.5 scrub ok 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: 10.7 scrub starts 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: 10.7 scrub ok 2024-08-17T17:48:21.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:20 smithi175 ceph-mon[101434]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:48:21.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.247 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:21.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:21.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:21.370 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:21.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:21.495 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:21.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:21.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:21.722 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.762 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:21.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:21.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:21.902 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:21.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[104516]: 10.15 scrub starts 2024-08-17T17:48:21.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[104516]: 10.15 scrub ok 2024-08-17T17:48:21.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[104516]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:48:21.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[104516]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:21.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[104516]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[109246]: 10.15 scrub starts 2024-08-17T17:48:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[109246]: 10.15 scrub ok 2024-08-17T17:48:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[109246]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:48:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[109246]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:21 smithi064 ceph-mon[109246]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:22.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:22.035 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:22.137 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:22.158 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:21 smithi175 ceph-mon[101434]: 10.15 scrub starts 2024-08-17T17:48:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:21 smithi175 ceph-mon[101434]: 10.15 scrub ok 2024-08-17T17:48:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:21 smithi175 ceph-mon[101434]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:48:22.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:21 smithi175 ceph-mon[101434]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:22.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:21 smithi175 ceph-mon[101434]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 475'13389 2024-08-17T17:48:22.259 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:22.281 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:22.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:22.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:22.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:48:22.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:22.615 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:22.644 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:22.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:22.788 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:22.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:22.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:23.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:22 smithi064 ceph-mon[109246]: pgmap v1498: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 106 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 855/82122 objects misplaced (1.041%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:48:23.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:22 smithi064 ceph-mon[109246]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:48:23.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:22 smithi064 ceph-mon[104516]: pgmap v1498: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 106 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 855/82122 objects misplaced (1.041%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:48:23.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:22 smithi064 ceph-mon[104516]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:48:23.078 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:22 smithi175 ceph-mon[101434]: pgmap v1498: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 106 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 855/82122 objects misplaced (1.041%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:48:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:22 smithi175 ceph-mon[101434]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:48:23.181 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.218 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:23.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:23.319 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:23.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:23.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:23.455 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:23.494 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:23.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.655 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:23.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:23.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.959 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:23.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:24.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:24.092 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:24.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:23 smithi175 ceph-mon[101434]: 10.17 scrub starts 2024-08-17T17:48:24.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:23 smithi064 ceph-mon[109246]: 10.17 scrub starts 2024-08-17T17:48:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:23] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-08-17T17:48:24.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:23 smithi064 ceph-mon[104516]: 10.17 scrub starts 2024-08-17T17:48:24.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.241 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:24.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:24.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:24.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.383 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:24.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:24.484 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.513 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:24.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:24.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.653 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:24.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:24.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:24.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.807 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:24.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:24.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:24.952 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:24.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:25.054 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:25.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:24 smithi064 ceph-mon[104516]: 10.17 scrub ok 2024-08-17T17:48:25.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:24 smithi064 ceph-mon[104516]: pgmap v1501: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/81585 objects misplaced (1.048%) 2024-08-17T17:48:25.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:24 smithi064 ceph-mon[109246]: 10.17 scrub ok 2024-08-17T17:48:25.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:24 smithi064 ceph-mon[109246]: pgmap v1501: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/81585 objects misplaced (1.048%) 2024-08-17T17:48:25.084 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:24 smithi175 ceph-mon[101434]: 10.17 scrub ok 2024-08-17T17:48:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:24 smithi175 ceph-mon[101434]: pgmap v1501: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/81585 objects misplaced (1.048%) 2024-08-17T17:48:25.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:25.230 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:25.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:25.376 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:25.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:25.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:25.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:25.620 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:25.658 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:25.759 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:25.795 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:25.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:25.927 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:26.065 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:26.095 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pg_num to 36 2024-08-17T17:48:26.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1005795026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:25 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3421737443' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:25 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:25 smithi175 ceph-mon[101434]: 10.14 scrub starts 2024-08-17T17:48:26.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1005795026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3421737443' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[109246]: 10.14 scrub starts 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1005795026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3421737443' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:48:26.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:25 smithi064 ceph-mon[104516]: 10.14 scrub starts 2024-08-17T17:48:26.200 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:26.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:26.339 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:26.441 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:26.473 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:26.574 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:26.606 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:26.707 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:26.747 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:26.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:27.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:27.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:27.058 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:48:27.058 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:48:27.058 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:48:27.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:48:27.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2024-08-17T17:48:27.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:27.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:27.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:27.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:27.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[104516]: pgmap v1502: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/79008 objects misplaced (1.082%) 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[104516]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[104516]: 10.14 scrub ok 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[104516]: 10.4 scrub starts 2024-08-17T17:48:27.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[109246]: pgmap v1502: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/79008 objects misplaced (1.082%) 2024-08-17T17:48:27.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:48:27.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[109246]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:48:27.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[109246]: 10.14 scrub ok 2024-08-17T17:48:27.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:26 smithi064 ceph-mon[109246]: 10.4 scrub starts 2024-08-17T17:48:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:26 smithi175 ceph-mon[101434]: pgmap v1502: 109 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 107 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 855/79008 objects misplaced (1.082%) 2024-08-17T17:48:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:26 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:48:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:26 smithi175 ceph-mon[101434]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:48:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:26 smithi175 ceph-mon[101434]: 10.14 scrub ok 2024-08-17T17:48:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:26 smithi175 ceph-mon[101434]: 10.4 scrub starts 2024-08-17T17:48:27.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:27.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:27.317 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:27.419 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:27.451 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:27.553 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:27.585 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:27.587 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:48:27.587 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:48:27.587 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2024-08-17T17:48:27.587 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2024-08-17T17:48:27.588 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:48:27.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:27.725 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:27.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:27.870 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:27.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[104516]: 10.4 scrub ok 2024-08-17T17:48:27.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:48:27.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:48:27.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[109246]: 10.4 scrub ok 2024-08-17T17:48:27.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:48:27.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:48:27.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:28.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:28.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:28.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:28.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:27 smithi175 ceph-mon[101434]: 10.4 scrub ok 2024-08-17T17:48:28.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:48:28.163 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T17:48:28.177 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:28.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:28.301 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:28.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:28.403 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:28.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:28.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:28.562 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:28.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:28.708 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:28.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.858 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:28.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:28.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:29.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:29.102 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:29.130 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: pgmap v1504: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/72009 objects misplaced (0.455%) 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:48:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:28 smithi175 ceph-mon[101434]: 10.2 scrub starts 2024-08-17T17:48:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: pgmap v1504: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/72009 objects misplaced (0.455%) 2024-08-17T17:48:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:48:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:48:29.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[104516]: 10.2 scrub starts 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: pgmap v1504: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/72009 objects misplaced (0.455%) 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:48:29.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:28 smithi064 ceph-mon[109246]: 10.2 scrub starts 2024-08-17T17:48:29.233 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.259 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.360 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.383 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:29.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:29.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.508 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:29.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:29.609 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.631 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:29.755 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:29.857 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.879 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:29.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:30.004 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:30.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:30.128 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:30.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.2 scrub ok 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.0 scrub starts 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.d scrub starts 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.0 scrub ok 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.b scrub starts 2024-08-17T17:48:30.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:29 smithi175 ceph-mon[101434]: 10.b scrub ok 2024-08-17T17:48:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.2 scrub ok 2024-08-17T17:48:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.0 scrub starts 2024-08-17T17:48:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.d scrub starts 2024-08-17T17:48:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.0 scrub ok 2024-08-17T17:48:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.b scrub starts 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[104516]: 10.b scrub ok 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.2 scrub ok 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.0 scrub starts 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.d scrub starts 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.0 scrub ok 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.b scrub starts 2024-08-17T17:48:30.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:29 smithi064 ceph-mon[109246]: 10.b scrub ok 2024-08-17T17:48:30.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:30.267 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:30.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.410 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.511 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.549 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:30.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:30.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.690 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:30.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:30.820 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:30.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:30.964 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:30.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:31.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:31.111 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:31.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:31.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.d scrub ok 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: pgmap v1507: 119 pgs: 2 peering, 8 unknown, 1 active+remapped+backfilling, 108 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/63984 objects misplaced (0.513%) 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.1 scrub starts 2024-08-17T17:48:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:30 smithi175 ceph-mon[101434]: 10.1 scrub ok 2024-08-17T17:48:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.d scrub ok 2024-08-17T17:48:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: pgmap v1507: 119 pgs: 2 peering, 8 unknown, 1 active+remapped+backfilling, 108 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/63984 objects misplaced (0.513%) 2024-08-17T17:48:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.1 scrub starts 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[104516]: 10.1 scrub ok 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.d scrub ok 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: pgmap v1507: 119 pgs: 2 peering, 8 unknown, 1 active+remapped+backfilling, 108 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 328/63984 objects misplaced (0.513%) 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 480'13658 2024-08-17T17:48:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.1 scrub starts 2024-08-17T17:48:31.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:30 smithi064 ceph-mon[109246]: 10.1 scrub ok 2024-08-17T17:48:31.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:31.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:31.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:31.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:31.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:31.516 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:31.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.649 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:31.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:31.657 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:31.751 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:31.787 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:31.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:31.932 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:32.034 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.066 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:31 smithi175 ceph-mon[101434]: 10.1b scrub starts 2024-08-17T17:48:32.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:32.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:31 smithi064 ceph-mon[104516]: 10.1b scrub starts 2024-08-17T17:48:32.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:31 smithi064 ceph-mon[109246]: 10.1b scrub starts 2024-08-17T17:48:32.201 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:32.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:32.348 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:32.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:32.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.485 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:32.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:32.586 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.618 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:48:32.720 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.752 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:32.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:32 smithi064 ceph-mon[104516]: pgmap v1509: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 1755/59115 objects misplaced (2.969%) 2024-08-17T17:48:32.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:32 smithi064 ceph-mon[104516]: 10.1b scrub ok 2024-08-17T17:48:32.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:32 smithi064 ceph-mon[109246]: pgmap v1509: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 1755/59115 objects misplaced (2.969%) 2024-08-17T17:48:32.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:32 smithi064 ceph-mon[109246]: 10.1b scrub ok 2024-08-17T17:48:32.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:33.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:33.063 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:33.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:32 smithi175 ceph-mon[101434]: pgmap v1509: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 1755/59115 objects misplaced (2.969%) 2024-08-17T17:48:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:32 smithi175 ceph-mon[101434]: 10.1b scrub ok 2024-08-17T17:48:33.165 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:33.202 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:33.304 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:33.342 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:33.443 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:33.481 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:33.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:33.621 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:33.722 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.759 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:33.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:33.860 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.895 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:33.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:33.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:34.034 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:34.119 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:33] "GET /metrics HTTP/1.1" 200 36702 "" "Prometheus/2.51.0" 2024-08-17T17:48:34.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:34.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:34.261 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:34.277 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:34.306 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:34.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:34.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:34.466 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:34.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:34.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:34.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:34.765 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:34.867 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:34.900 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:34.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:35.002 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.037 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:35.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:35.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:34 smithi175 ceph-mon[101434]: pgmap v1510: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/58035 objects misplaced (3.024%) 2024-08-17T17:48:35.185 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:34 smithi064 ceph-mon[104516]: pgmap v1510: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/58035 objects misplaced (3.024%) 2024-08-17T17:48:35.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:34 smithi064 ceph-mon[109246]: pgmap v1510: 119 pgs: 1 active+remapped+backfilling, 2 peering, 2 unknown, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/58035 objects misplaced (3.024%) 2024-08-17T17:48:35.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.321 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:35.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:35.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:35.461 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:35.563 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.607 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:35.709 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:35.741 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:35.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:35.881 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:35.914 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:48:35.983 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:36.013 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:36.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:36.152 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:36.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:35 smithi175 ceph-mon[101434]: 10.c scrub starts 2024-08-17T17:48:36.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:35 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:48:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/696589473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:35 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[104516]: 10.c scrub starts 2024-08-17T17:48:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:48:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/696589473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[109246]: 10.c scrub starts 2024-08-17T17:48:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"} : dispatch 2024-08-17T17:48:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/696589473' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:35 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:36.254 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:36.282 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:36.383 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:36.409 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:36.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:36.533 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:36.596 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:48:36.596 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:36.596 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-17T17:48:36.596 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:48:36.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:48:36.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:36.657 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:36.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:36.781 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:36.882 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:36.897 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:36 smithi175 systemd[1]: Stopping Ceph osd.7 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:48:36.910 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:37.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: pgmap v1511: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/51798 objects misplaced (3.388%) 2024-08-17T17:48:37.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:37.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:48:37.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:48:37.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: 10.22 scrub starts 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[104516]: 10.22 scrub ok 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: pgmap v1511: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/51798 objects misplaced (3.388%) 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: 10.22 scrub starts 2024-08-17T17:48:37.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:36 smithi064 ceph-mon[109246]: 10.22 scrub ok 2024-08-17T17:48:37.054 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.156 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:37.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: pgmap v1511: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 1755/51798 objects misplaced (3.388%) 2024-08-17T17:48:37.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "27"}]': finished 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: 10.22 scrub starts 2024-08-17T17:48:37.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-mon[101434]: 10.22 scrub ok 2024-08-17T17:48:37.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[56879]: 2024-08-17T17:48:36.937+0000 7ff5a9c90640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:48:37.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[56879]: 2024-08-17T17:48:36.937+0000 7ff5a9c90640 -1 osd.7 485 *** Got signal Terminated *** 2024-08-17T17:48:37.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[56879]: 2024-08-17T17:48:36.937+0000 7ff5a9c90640 -1 osd.7 485 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:48:37.183 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:37.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.309 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.312 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:37.329 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:48:37.329 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2024-08-17T17:48:37.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2024-08-17T17:48:37.329 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241823 8.c 8:35e3d2d5:::benchmark_data_smithi064_242541_object119330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.455640+0000", 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.87057769600005, 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.173862068, 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241823 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455640+0000", 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455640+0000", 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455641+0000", 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5030000000000001e-06 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455646+0000", 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9230000000000001e-06 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455658+0000", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2176e-05 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455665+0000", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8870000000000003e-06 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455763+0000", 2024-08-17T17:48:37.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7709999999999998e-05 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.455773+0000", 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9499999999999996e-06 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.394608+0000", 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93883561599999998 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.394623+0000", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.455e-05 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.621399+0000", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2267756480000001 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.621550+0000", 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015114699999999999 2024-08-17T17:48:37.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.621944+0000", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039380099999999999 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.628830+0000", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068869070000000003 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.628997+0000", 2024-08-17T17:48:37.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000166847 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629042+0000", 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5059999999999999e-05 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629082+0000", 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9480000000000001e-05 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629403+0000", 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032080699999999999 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629438+0000", 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5487000000000001e-05 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629502+0000", 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3569000000000001e-05 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241826 8.c 8:363a8582:::benchmark_data_smithi064_242541_object119333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.456898+0000", 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.86931960599998, 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.172762225, 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241826 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456898+0000", 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456898+0000", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456899+0000", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.158e-06 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456903+0000", 2024-08-17T17:48:37.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5229999999999997e-06 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456915+0000", 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1094000000000001e-05 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456922+0000", 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5440000000000001e-06 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456974+0000", 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2278999999999999e-05 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.456984+0000", 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8120000000000002e-06 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.395504+0000", 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93851982300000003 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.395517+0000", 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3249e-05 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.624323+0000", 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2288061159999999 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.624427+0000", 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010396899999999999 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.624636+0000", 2024-08-17T17:48:37.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020915699999999999 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629095+0000", 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044583299999999999 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629132+0000", 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7669000000000001e-05 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629165+0000", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2400000000000001e-05 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629199+0000", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3730999999999999e-05 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629547+0000", 2024-08-17T17:48:37.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034824299999999998 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629583+0000", 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6202000000000003e-05 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629660+0000", 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6926000000000002e-05 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241830 8.c 8:355a1485:::benchmark_data_smithi064_242541_object119337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.460080+0000", 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.86613765499999, 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.169779144, 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241830 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.349 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460080+0000", 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460080+0000", 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460081+0000", 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.141e-06 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460086+0000", 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.118e-06 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460102+0000", 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.645e-05 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460111+0000", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2530000000000004e-06 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460230+0000", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011948 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.460249+0000", 2024-08-17T17:48:37.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8406e-05 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396010+0000", 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93576121800000001 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396020+0000", 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0686999999999999e-05 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.626774+0000", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.230753585 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.626858+0000", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3664000000000004e-05 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.627178+0000", 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032007800000000001 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629100+0000", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001921807 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629237+0000", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013745599999999999 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629291+0000", 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3477999999999999e-05 2024-08-17T17:48:37.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629330+0000", 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9202999999999997e-05 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629713+0000", 2024-08-17T17:48:37.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038364999999999997 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629745+0000", 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1279000000000003e-05 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.629859+0000", 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114191 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.357 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241848 8.c 8:37bdc519:::benchmark_data_smithi064_242541_object119355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.473409+0000", 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.85280830900001, 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.171703698, 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.358 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241848 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473409+0000", 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473409+0000", 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473410+0000", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3549999999999999e-06 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473415+0000", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1949999999999996e-06 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473426+0000", 2024-08-17T17:48:37.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1134999999999999e-05 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473433+0000", 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.007e-06 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473499+0000", 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6309000000000005e-05 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.473510+0000", 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0988000000000001e-05 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396217+0000", 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92270701099999997 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396227+0000", 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5170000000000005e-06 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.630012+0000", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.23378538 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.630084+0000", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1691000000000002e-05 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.630332+0000", 2024-08-17T17:48:37.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024822700000000001 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638261+0000", 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079291899999999992 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638277+0000", 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6257e-05 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638320+0000", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3204999999999997e-05 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638391+0000", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0842999999999999e-05 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.644980+0000", 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065883130000000002 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645020+0000", 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0706e-05 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645113+0000", 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2368999999999996e-05 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.366 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241891 8.c 8:349b19e7:::benchmark_data_smithi064_242541_object119398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.507466+0000", 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.81875143100001, 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1379191400000002, 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241891 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507466+0000", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507466+0000", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507468+0000", 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.64e-06 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507473+0000", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5099999999999998e-06 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507484+0000", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1398e-05 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507492+0000", 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8199999999999997e-06 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507546+0000", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4216999999999999e-05 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.507557+0000", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0484e-05 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396256+0000", 2024-08-17T17:48:37.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88869927100000001 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396264+0000", 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1310000000000006e-06 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.632890+0000", 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2366251699999999 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.633010+0000", 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120693 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.633400+0000", 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039012999999999998 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638406+0000", 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050052159999999998 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638457+0000", 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1344999999999997e-05 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638508+0000", 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1490000000000003e-05 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638568+0000", 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9842000000000001e-05 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645279+0000", 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067108840000000003 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645314+0000", 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5330000000000002e-05 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645385+0000", 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0568999999999995e-05 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241929 8.c 8:302cd086:::benchmark_data_smithi064_242541_object119436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.537456+0000", 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.78876084700005, 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1080976059999998, 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.375 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241929 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537456+0000", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537456+0000", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537457+0000", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2900000000000002e-07 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537461+0000", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9509999999999999e-06 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537471+0000", 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7219999999999994e-06 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537476+0000", 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7509999999999999e-06 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537527+0000", 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0909000000000003e-05 2024-08-17T17:48:37.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.537533+0000", 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4180000000000002e-06 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396289+0000", 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85875633600000001 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396297+0000", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.588e-06 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.635392+0000", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2390949959999999 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.635505+0000", 2024-08-17T17:48:37.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112998 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.635781+0000", 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000276366 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638675+0000", 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028936119999999998 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638741+0000", 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5970999999999996e-05 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638761+0000", 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0264000000000001e-05 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638802+0000", 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0748999999999999e-05 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645426+0000", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066242540000000004 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645466+0000", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9424000000000001e-05 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645554+0000", 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8467999999999994e-05 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241975 8.c 8:36929748:::benchmark_data_smithi064_242541_object119482:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.584458+0000", 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.74175972800003, 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0614159060000001, 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241975 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584458+0000", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584458+0000", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584459+0000", 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.032e-06 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584463+0000", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2660000000000003e-06 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584475+0000", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.201e-05 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584483+0000", 2024-08-17T17:48:37.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6839999999999996e-06 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584541+0000", 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8034e-05 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.584550+0000", 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5240000000000006e-06 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396322+0000", 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81177225500000005 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396331+0000", 2024-08-17T17:48:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6400000000000003e-06 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638844+0000", 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.242513151 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.638990+0000", 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014596699999999999 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.639446+0000", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045592700000000001 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645157+0000", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057105799999999998 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645170+0000", 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3525e-05 2024-08-17T17:48:37.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645204+0000", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4029000000000002e-05 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645240+0000", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6202999999999999e-05 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645631+0000", 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039065900000000001 2024-08-17T17:48:37.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645732+0000", 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101105 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.645874+0000", 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014131500000000001 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:241984 8.c 8:37d76503:::benchmark_data_smithi064_242541_object119491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.592351+0000", 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.73386596399996, 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.063004088, 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 241984 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.394 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592351+0000", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592351+0000", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592353+0000", 2024-08-17T17:48:37.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.158e-06 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592357+0000", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4619999999999998e-06 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592367+0000", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0505e-05 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592375+0000", 2024-08-17T17:48:37.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1250000000000004e-06 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592499+0000", 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012415399999999999 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.592509+0000", 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0519e-05 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396359+0000", 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80385008400000002 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396367+0000", 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9950000000000005e-06 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.646068+0000", 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.249700689 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.646185+0000", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116458 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.646710+0000", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052538699999999999 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655016+0000", 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083059980000000002 2024-08-17T17:48:37.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655028+0000", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1759e-05 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655073+0000", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5130000000000003e-05 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655164+0000", 2024-08-17T17:48:37.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1439000000000003e-05 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655200+0000", 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5343999999999999e-05 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655247+0000", 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6940999999999997e-05 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655355+0000", 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108941 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242005 8.c 8:34de0241:::benchmark_data_smithi064_242541_object119512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.609115+0000", 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.71710214999996, 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0652192180000002, 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.402 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242005 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609115+0000", 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609115+0000", 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609117+0000", 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.553e-06 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609123+0000", 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9479999999999996e-06 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609135+0000", 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2777e-05 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609143+0000", 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8949999999999998e-06 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609204+0000", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1036000000000003e-05 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.609215+0000", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0288999999999999e-05 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396393+0000", 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78717850700000003 2024-08-17T17:48:37.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396401+0000", 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4209999999999998e-06 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.649540+0000", 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2531397440000001 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.649677+0000", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136651 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.650129+0000", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045164100000000002 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655090+0000", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049618099999999997 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661090+0000", 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059991259999999996 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661212+0000", 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012277999999999999 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661253+0000", 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0405999999999999e-05 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674118+0000", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012865359 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674192+0000", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.347e-05 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674334+0000", 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142805 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.410 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242014 8.c 8:3603b303:::benchmark_data_smithi064_242541_object119521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.613779+0000", 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.71243874200002, 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0640051939999999, 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242014 2024-08-17T17:48:37.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613779+0000", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613779+0000", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613779+0000", 2024-08-17T17:48:37.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5899999999999995e-07 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613784+0000", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4680000000000003e-06 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613793+0000", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2909999999999996e-06 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613798+0000", 2024-08-17T17:48:37.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4440000000000001e-06 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613860+0000", 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2428000000000006e-05 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.613870+0000", 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396426+0000", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78255591999999996 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396433+0000", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6340000000000001e-06 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.652345+0000", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.255911961 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.652428+0000", 2024-08-17T17:48:37.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2570000000000004e-05 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.652703+0000", 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000275584 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655367+0000", 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002663168 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661156+0000", 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057890540000000001 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661329+0000", 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173079 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667530+0000", 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062009700000000001 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677642+0000", 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010112374 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677679+0000", 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7159000000000001e-05 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677784+0000", 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010451299999999999 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242036 8.c 8:3640d452:::benchmark_data_smithi064_242541_object119543:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.634296+0000", 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.69192153200004, 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0436780379999999, 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.419 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242036 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634296+0000", 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634296+0000", 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634297+0000", 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.283e-06 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634312+0000", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5082000000000001e-05 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634338+0000", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5695999999999999e-05 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634346+0000", 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9279999999999993e-06 2024-08-17T17:48:37.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634445+0000", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9117999999999995e-05 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.634458+0000", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2609000000000001e-05 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396458+0000", 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76200088300000002 2024-08-17T17:48:37.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396466+0000", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3429999999999998e-06 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655420+0000", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2589538309999999 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.655603+0000", 2024-08-17T17:48:37.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018361500000000001 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.656000+0000", 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039684699999999998 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661168+0000", 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051680859999999997 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661291+0000", 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012298800000000001 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661372+0000", 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0829999999999997e-05 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667674+0000", 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063016010000000004 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677846+0000", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010172601 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677880+0000", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4187999999999999e-05 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677974+0000", 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3509000000000002e-05 2024-08-17T17:48:37.427 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242098 8.c 8:353bbecc:::benchmark_data_smithi064_242541_object119605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.693636+0000", 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.63258094399998, 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9917077170000002, 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.428 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242098 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693636+0000", 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693636+0000", 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693638+0000", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9099999999999999e-06 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693644+0000", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.626e-06 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693662+0000", 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7588000000000002e-05 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693670+0000", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0940000000000008e-06 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693735+0000", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5189000000000002e-05 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.693746+0000", 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0917000000000001e-05 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396492+0000", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70274621000000004 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396501+0000", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7029999999999995e-06 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.658199+0000", 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2616986370000001 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.658317+0000", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011807300000000001 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.658739+0000", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042119899999999998 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667418+0000", 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086799080000000001 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667473+0000", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.435e-05 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667574+0000", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010111399999999999 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.673990+0000", 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064156839999999996 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685155+0000", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01116515 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685215+0000", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0503e-05 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685344+0000", 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012886199999999999 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242151 8.c 8:33feb9af:::benchmark_data_smithi064_242541_object119658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.753877+0000", 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.57234029899996, 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9318571599999999, 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242151 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753877+0000", 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753877+0000", 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753878+0000", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.95e-07 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753893+0000", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4960000000000001e-05 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753904+0000", 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0584000000000001e-05 2024-08-17T17:48:37.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753909+0000", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2920000000000003e-06 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753972+0000", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3102000000000007e-05 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.753983+0000", 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0555e-05 2024-08-17T17:48:37.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396527+0000", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64254425000000004 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396534+0000", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661406+0000", 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.264871528 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661515+0000", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108696 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.661872+0000", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035765200000000002 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667545+0000", 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056729149999999997 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667623+0000", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7883e-05 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667720+0000", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6401000000000001e-05 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674058+0000", 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063388979999999999 2024-08-17T17:48:37.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685546+0000", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01148777 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685596+0000", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0018000000000001e-05 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685734+0000", 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137971 2024-08-17T17:48:37.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242158 8.c 8:31eec56a:::benchmark_data_smithi064_242541_object119665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.762616+0000", 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.56360157500001, 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9234534249999999, 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.444 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242158 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762616+0000", 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762616+0000", 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762617+0000", 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5799999999999998e-07 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762621+0000", 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2740000000000001e-06 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762632+0000", 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1201e-05 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762637+0000", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1780000000000002e-06 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762694+0000", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7117000000000002e-05 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.762703+0000", 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.242e-06 2024-08-17T17:48:37.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396562+0000", 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.63385960900000005 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396571+0000", 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6000000000000007e-06 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.664214+0000", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2676428550000001 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.664326+0000", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111872 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.664774+0000", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044848400000000002 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667639+0000", 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002864765 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667765+0000", 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012559100000000001 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674562+0000", 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067979859999999998 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674737+0000", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000174146 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685907+0000", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011170566 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.685958+0000", 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0386999999999997e-05 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686069+0000", 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111594 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242169 8.c 8:354c0a4a:::benchmark_data_smithi064_242541_object119676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.781385+0000", 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.54483224199998, 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.904993261, 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.452 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242169 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781385+0000", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781385+0000", 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781387+0000", 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9609999999999999e-06 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781393+0000", 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6030000000000004e-06 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781409+0000", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6062999999999999e-05 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781416+0000", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2320000000000004e-06 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781478+0000", 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2226999999999999e-05 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.781487+0000", 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8100000000000004e-06 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396600+0000", 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61511329999999997 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.396609+0000", 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3280000000000006e-06 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667803+0000", 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2711944800000001 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.667896+0000", 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3102000000000004e-05 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.668187+0000", 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029077999999999998 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674350+0000", 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061631359999999996 2024-08-17T17:48:37.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674395+0000", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.481e-05 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674620+0000", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022493599999999999 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677550+0000", 2024-08-17T17:48:37.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029299970000000002 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686161+0000", 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086114669999999994 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686247+0000", 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5822000000000006e-05 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686378+0000", 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013120699999999999 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:242183 8.c 8:32e4d3a3:::benchmark_data_smithi064_242541_object119690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:10.821292+0000", 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.50492507599995, 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8653174799999999, 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.461 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 242183 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821292+0000", 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821292+0000", 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821293+0000", 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0049999999999999e-06 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821297+0000", 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8130000000000002e-06 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821308+0000", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0685e-05 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821315+0000", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6650000000000003e-06 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821376+0000", 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0086e-05 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:10.821384+0000", 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.397041+0000", 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.57565622699999996 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:11.397051+0000", 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0808999999999999e-05 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.670676+0000", 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2736242369999999 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.670776+0000", 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100765 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,5,7", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.671044+0000", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026720200000000002 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674459+0000", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034156299999999998 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674478+0000", 2024-08-17T17:48:37.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8947999999999999e-05 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.674686+0000", 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020772100000000001 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.677601+0000", 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029150130000000002 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686424+0000", 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088235069999999995 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686489+0000", 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4072000000000006e-05 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:12.686610+0000", 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121186 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.469 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:244354 8.c 8:302f23f0:::benchmark_data_smithi064_242541_object121861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:14.468501+0000", 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 562.85771653500001, 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99353901099999997, 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.470 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 244354 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468501+0000", 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468501+0000", 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468502+0000", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.054e-06 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468505+0000", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8959999999999999e-06 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468513+0000", 2024-08-17T17:48:37.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5839999999999995e-06 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468520+0000", 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1859999999999997e-06 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468588+0000", 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8645000000000004e-05 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468598+0000", 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5540000000000003e-06 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.457559+0000", 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98896128100000003 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.457694+0000", 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013524000000000001 2024-08-17T17:48:37.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.458100+0000", 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000406103 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.461933+0000", 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038329560000000002 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.461949+0000", 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5203e-05 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.461982+0000", 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3837000000000001e-05 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.462014+0000", 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1724000000000003e-05 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.462040+0000", 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5748000000000001e-05 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45488.0:244358 8.c 8:324afa47:::benchmark_data_smithi064_242541_object121865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:39:14.468800+0000", 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 562.85741728599999, 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99806248099999995, 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.477 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45488", 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2454128608", 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 244358 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468800+0000", 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468800+0000", 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468801+0000", 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000003e-07 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468805+0000", 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8850000000000001e-06 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468814+0000", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4749999999999999e-06 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468821+0000", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4500000000000001e-06 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468891+0000", 2024-08-17T17:48:37.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0625999999999998e-05 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:14.468900+0000", 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5180000000000004e-06 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.460383+0000", 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99148309700000004 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.460491+0000", 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010836900000000001 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.460729+0000", 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023787600000000001 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.466715+0000", 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059858450000000001 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.466727+0000", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.166e-05 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.466783+0000", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6048000000000003e-05 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.466818+0000", 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5018000000000003e-05 2024-08-17T17:48:37.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:39:15.466863+0000", 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4854000000000001e-05 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:204050 10.17 10:ed101f40:::benchmark_data_smithi064_306376_object79069:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e460)", 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:54.613967+0000", 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 42.712250419999997, 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.030164123, 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.484 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 204050 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613967+0000", 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613967+0000", 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613968+0000", 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2440000000000001e-06 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613971+0000", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0860000000000002e-06 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613980+0000", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.636e-06 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.613986+0000", 2024-08-17T17:48:37.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5910000000000001e-06 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.622839+0000", 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088522219999999999 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.622852+0000", 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3679000000000001e-05 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.644070+0000", 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.021217649 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.644131+0000", 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1016e-05 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.55666.0:204074 10.17 10:e939e9af:::benchmark_data_smithi064_306376_object79093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-08-17T17:48:37.489 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2024-08-17T17:47:54.628379+0000", 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 42.697838574000002, 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0273580200000001, 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "continuous": false, 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.55666", 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 204074 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2024-08-17T17:48:37.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628379+0000", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628379+0000", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628381+0000", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.897e-06 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2024-08-17T17:48:37.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628391+0000", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0380000000000001e-05 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628409+0000", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7754e-05 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628417+0000", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7139999999999993e-06 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628588+0000", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171514 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:54.628612+0000", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4049e-05 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.644521+0000", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.015908729 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.644643+0000", 2024-08-17T17:48:37.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012223299999999999 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.645071+0000", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042800200000000003 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.652132+0000", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070608310000000001 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.652153+0000", 2024-08-17T17:48:37.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1560999999999999e-05 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.655670+0000", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003516473 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.655697+0000", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7038000000000001e-05 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2024-08-17T17:47:55.655737+0000", 2024-08-17T17:48:37.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9845000000000002e-05 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout: } 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2024-08-17T17:48:37.496 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:48:37.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:37.518 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:37.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.642 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:37.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.890 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:38.020 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:38.122 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:38.146 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.11 scrub starts 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.11 scrub ok 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.1c scrub starts 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.1c scrub ok 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:48:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: osd.7 marked itself down and dead 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.1a scrub starts 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: 10.1a scrub ok 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.11 scrub starts 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.11 scrub ok 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.1c scrub starts 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.1c scrub ok 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: osd.7 marked itself down and dead 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.1a scrub starts 2024-08-17T17:48:38.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:48:38.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: 10.1a scrub ok 2024-08-17T17:48:38.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:37 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:38.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:38.269 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:38.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:38.393 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:38.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.11 scrub starts 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.11 scrub ok 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.1c scrub starts 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.1c scrub ok 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: osd.7 marked itself down and dead 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.1a scrub starts 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"} : dispatch 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: 10.1a scrub ok 2024-08-17T17:48:38.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:37 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:38.495 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:38.523 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:38.625 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:38.653 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:38.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:48:38.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:38.805 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:38.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:38.929 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:39.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:39.053 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:39.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:39.160 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:38 smithi175 podman[243510]: 2024-08-17 17:48:38.819902758 +0000 UTC m=+1.986264709 container died 7ccae91f6ae282f2e29f523560088ee87c2f931725c3f8ffc99f64be5e4f8b3b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 858 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138/41037 objects misplaced (0.336%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.c scrub starts 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.c scrub ok 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: osdmap e486: 8 total, 7 up, 8 in 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.a scrub starts 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.a scrub ok 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.e scrub starts 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: 10.e scrub ok 2024-08-17T17:48:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:38 smithi175 ceph-mon[101434]: osdmap e487: 8 total, 7 up, 8 in 2024-08-17T17:48:39.175 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 858 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138/41037 objects misplaced (0.336%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:48:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.c scrub starts 2024-08-17T17:48:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.c scrub ok 2024-08-17T17:48:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:48:39.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: osdmap e486: 8 total, 7 up, 8 in 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.a scrub starts 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.a scrub ok 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.e scrub starts 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: 10.e scrub ok 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[104516]: osdmap e487: 8 total, 7 up, 8 in 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: pgmap v1514: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 858 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138/41037 objects misplaced (0.336%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:48:39.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.c scrub starts 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.c scrub ok 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "28"}]': finished 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: osdmap e486: 8 total, 7 up, 8 in 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.a scrub starts 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.a scrub ok 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.e scrub starts 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: 10.e scrub ok 2024-08-17T17:48:39.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:38 smithi064 ceph-mon[109246]: osdmap e487: 8 total, 7 up, 8 in 2024-08-17T17:48:39.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:39.300 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:39.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:39.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:39.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:39.525 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:39.574 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:39 smithi175 podman[243510]: 2024-08-17 17:48:39.381074443 +0000 UTC m=+2.547436429 container cleanup 7ccae91f6ae282f2e29f523560088ee87c2f931725c3f8ffc99f64be5e4f8b3b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:48:39.575 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:39 smithi175 bash[243510]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:48:39.649 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:39.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:39.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:39.777 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:48:39.777 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:39.780 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:48:39.780 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:39.796 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:39.797 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:48:39.798 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:39.814 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:48:39.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:39.830 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:48:39.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:39.846 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:48:39.847 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:39.859 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:39 smithi175 podman[243764]: 2024-08-17 17:48:39.574390163 +0000 UTC m=+0.937781349 container remove 7ccae91f6ae282f2e29f523560088ee87c2f931725c3f8ffc99f64be5e4f8b3b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:48:39.863 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:48:39.864 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:39.864 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:48:39.898 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:40.046 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:40.148 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.200 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:40.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.301 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:40.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.348 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:40.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:40.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:40.488 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:40.590 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.634 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:40.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:40.733 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:40.736 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.769 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:40.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:40.870 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: 10.12 scrub starts 2024-08-17T17:48:40.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: pgmap v1517: 119 pgs: 2 active+undersized, 3 peering, 12 stale+active+clean, 102 active+clean; 760 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 7.3 MiB/s, 113 objects/s recovering 2024-08-17T17:48:40.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: 10.12 scrub ok 2024-08-17T17:48:40.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: osdmap e488: 8 total, 7 up, 8 in 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: 10.1b starting backfill to osd.3 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: 10.1b starting backfill to osd.4 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1507267439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: 10.12 scrub starts 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: pgmap v1517: 119 pgs: 2 active+undersized, 3 peering, 12 stale+active+clean, 102 active+clean; 760 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 7.3 MiB/s, 113 objects/s recovering 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: 10.12 scrub ok 2024-08-17T17:48:40.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: osdmap e488: 8 total, 7 up, 8 in 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: 10.1b starting backfill to osd.3 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: 10.1b starting backfill to osd.4 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1507267439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:40.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:40.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:40.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: 10.12 scrub starts 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: pgmap v1517: 119 pgs: 2 active+undersized, 3 peering, 12 stale+active+clean, 102 active+clean; 760 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 7.3 MiB/s, 113 objects/s recovering 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: 10.12 scrub ok 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: osdmap e488: 8 total, 7 up, 8 in 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: 10.1b starting backfill to osd.3 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: 10.1b starting backfill to osd.4 from (0'0,0'0] MAX to 485'14585 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1507267439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:48:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:40.912 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:41.003 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:41.043 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:41.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:41.182 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:41.223 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:41.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:48:41.284 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.353 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:41.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:41.454 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:41.492 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:41 smithi175 podman[244138]: 2024-08-17 17:48:41.086508979 +0000 UTC m=+0.034938667 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:48:41.492 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:41 smithi175 podman[244138]: 2024-08-17 17:48:41.218638433 +0000 UTC m=+0.167068114 container create 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:48:41.492 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:41 smithi175 podman[244138]: 2024-08-17 17:48:41.490780442 +0000 UTC m=+0.439210137 container init 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:48:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:41.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:41.596 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:41.633 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:41.729 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:41.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:41.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:41.812 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:41 smithi175 ceph-mon[101434]: Health check failed: Degraded data redundancy: 1878/32475 objects degraded (5.783%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:41.812 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:41 smithi175 podman[244138]: 2024-08-17 17:48:41.502814443 +0000 UTC m=+0.451244127 container start 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-ced649f, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:48:41.812 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:41 smithi175 podman[244138]: 2024-08-17 17:48:41.613678203 +0000 UTC m=+0.562107886 container attach 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:48:41.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:41.906 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:41.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:41 smithi064 ceph-mon[104516]: Health check failed: Degraded data redundancy: 1878/32475 objects degraded (5.783%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:41.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:41 smithi064 ceph-mon[109246]: Health check failed: Degraded data redundancy: 1878/32475 objects degraded (5.783%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:42.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:42.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:42.069 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:42 smithi175 podman[244138]: 2024-08-17 17:48:42.043731702 +0000 UTC m=+0.992161383 container died 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:48:42.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:42.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.181 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:42.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:42.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:42.322 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:42.424 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:42.469 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:42.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:42.624 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[104516]: pgmap v1519: 119 pgs: 15 active+undersized, 3 peering, 3 stale+active+clean, 16 active+undersized+degraded, 82 active+clean; 679 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 1878/32475 objects degraded (5.783%) 2024-08-17T17:48:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[104516]: 10.f scrub starts 2024-08-17T17:48:42.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[104516]: 10.f scrub ok 2024-08-17T17:48:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[109246]: pgmap v1519: 119 pgs: 15 active+undersized, 3 peering, 3 stale+active+clean, 16 active+undersized+degraded, 82 active+clean; 679 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 1878/32475 objects degraded (5.783%) 2024-08-17T17:48:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[109246]: 10.f scrub starts 2024-08-17T17:48:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:42 smithi064 ceph-mon[109246]: 10.f scrub ok 2024-08-17T17:48:42.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:42.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:42.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:42.850 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:42 smithi175 ceph-mon[101434]: pgmap v1519: 119 pgs: 15 active+undersized, 3 peering, 3 stale+active+clean, 16 active+undersized+degraded, 82 active+clean; 679 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 1878/32475 objects degraded (5.783%) 2024-08-17T17:48:42.850 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:42 smithi175 ceph-mon[101434]: 10.f scrub starts 2024-08-17T17:48:42.850 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:42 smithi175 ceph-mon[101434]: 10.f scrub ok 2024-08-17T17:48:42.850 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:48:42 smithi175 podman[244315]: 2024-08-17 17:48:42.738955844 +0000 UTC m=+0.914470288 container remove 12781013288c1f56dc7ae2c3f598f56c7dbb99921d2146be20d941d2679aab8b (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:48:42.887 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:42.915 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.017 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:43.053 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:43.155 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.188 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.290 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:43.326 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:43.427 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.463 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.565 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.595 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.696 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:48:43.697 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.755 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:43.785 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:48:43.786 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:48:43.790 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:43.926 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:44.028 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:44.051 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:44.153 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:44.174 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:43] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-08-17T17:48:44.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:44.299 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:44.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.423 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:44.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:44.517 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2024-08-17T17:48:44.524 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:44.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.547 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:44.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:44.648 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:44.681 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:44.755 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 22 active+undersized, 3 peering, 18 active+undersized+degraded, 75 active+clean; 632 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2107/30177 objects degraded (6.982%); 293/30177 objects misplaced (0.971%) 2024-08-17T17:48:44.755 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: osdmap e489: 8 total, 7 up, 8 in 2024-08-17T17:48:44.755 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3654136490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.755 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.755 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: 10.1d scrub starts 2024-08-17T17:48:44.756 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:44 smithi175 ceph-mon[101434]: 10.1d scrub ok 2024-08-17T17:48:44.783 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.820 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:44.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 22 active+undersized, 3 peering, 18 active+undersized+degraded, 75 active+clean; 632 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2107/30177 objects degraded (6.982%); 293/30177 objects misplaced (0.971%) 2024-08-17T17:48:44.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: osdmap e489: 8 total, 7 up, 8 in 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3654136490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: 10.1d scrub starts 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[104516]: 10.1d scrub ok 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: pgmap v1520: 119 pgs: 1 active+remapped+backfilling, 22 active+undersized, 3 peering, 18 active+undersized+degraded, 75 active+clean; 632 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2107/30177 objects degraded (6.982%); 293/30177 objects misplaced (0.971%) 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: osdmap e489: 8 total, 7 up, 8 in 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3654136490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: 10.1d scrub starts 2024-08-17T17:48:44.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:44 smithi064 ceph-mon[109246]: 10.1d scrub ok 2024-08-17T17:48:44.917 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:44.917 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:44.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:44.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:45.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:45.096 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:45.197 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:45.231 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:45.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:45.371 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:45.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:45.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.517 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:45.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:45.617 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:45.635 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:48:45.654 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:45.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:45.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:45.760 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:45 smithi175 ceph-mon[101434]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:45.760 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:45 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:45.760 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:45 smithi175 ceph-mon[101434]: osdmap e490: 8 total, 7 up, 8 in 2024-08-17T17:48:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.813 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:45.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:45.914 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[104516]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:45.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[104516]: osdmap e490: 8 total, 7 up, 8 in 2024-08-17T17:48:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[109246]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:45.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:45 smithi064 ceph-mon[109246]: osdmap e490: 8 total, 7 up, 8 in 2024-08-17T17:48:45.964 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:45.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:46.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.106 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:46.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:46.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:46.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:46.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.395 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:46.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:46.473 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:46.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:46.538 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:46.640 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:46.676 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:46 smithi064 ceph-mon[104516]: pgmap v1523: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 522 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2312/24915 objects degraded (9.280%); 207/24915 objects misplaced (0.831%) 2024-08-17T17:48:46.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:46 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:48:46.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:46 smithi064 ceph-mon[109246]: pgmap v1523: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 522 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2312/24915 objects degraded (9.280%); 207/24915 objects misplaced (0.831%) 2024-08-17T17:48:46.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:46 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:48:46.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.805 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:46 smithi175 ceph-mon[101434]: pgmap v1523: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 522 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 12 KiB/s rd, 5 op/s; 2312/24915 objects degraded (9.280%); 207/24915 objects misplaced (0.831%) 2024-08-17T17:48:46.805 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:46 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:48:46.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:46.915 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:46.946 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:48:47.018 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:47.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:47.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:48:47.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:47.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:48:47.091 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:47.130 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:48:47.130 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:47.169 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:48:47.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:47.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:48:47.205 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:47.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:47.240 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:48:47.241 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:47.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:48:47.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:47.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:47.383 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:47.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.527 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:47.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:47.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:48:47.628 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:47.660 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:47.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:47 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/802290745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.814 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:47.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:47.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:47 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/802290745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:47.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:47 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/802290745' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:47.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.955 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:47.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:48.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:48.101 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:48.183 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:48.203 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:48.234 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:48.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.368 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.469 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.495 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:48.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.597 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:48.623 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:48.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:48.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:48.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:48.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:48.786 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 6, 7, 4, 5, 3] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:48:48.786 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:48.787 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 6, 7, 4, 5, 3] 2024-08-17T17:48:48.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd out 6 2024-08-17T17:48:48.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:48 smithi064 ceph-mon[109246]: pgmap v1524: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 337 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 9.2 KiB/s rd, 4 op/s; 1246/16026 objects degraded (7.775%); 207/16026 objects misplaced (1.292%) 2024-08-17T17:48:48.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:48 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 1246/16026 objects degraded (7.775%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:48.788 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:48 smithi064 ceph-mon[104516]: pgmap v1524: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 337 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 9.2 KiB/s rd, 4 op/s; 1246/16026 objects degraded (7.775%); 207/16026 objects misplaced (1.292%) 2024-08-17T17:48:48.788 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:48 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 1246/16026 objects degraded (7.775%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:48.876 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 124978 objects 2024-08-17T17:48:48.876 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :144.054 2024-08-17T17:48:48.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:48.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.901 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:48.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:48.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:48 smithi175 ceph-mon[101434]: pgmap v1524: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 337 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 9.2 KiB/s rd, 4 op/s; 1246/16026 objects degraded (7.775%); 207/16026 objects misplaced (1.292%) 2024-08-17T17:48:48.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:48 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 1246/16026 objects degraded (7.775%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:49.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:49.024 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:49.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:49.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:49.148 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:49.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:49.272 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:49.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:49.397 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:49.499 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:49.520 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:49.621 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:49.646 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:49.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:49.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:49.893 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:49.995 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:50.018 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:50.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:50.150 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:50.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:50.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:50.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:50.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:50.401 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:50.503 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:50.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:50.627 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:50.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:50.652 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:48:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:48:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:48:50.652 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:48:50.652 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:48:50.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:50.774 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:50.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:50.899 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:50 smithi175 ceph-mon[101434]: pgmap v1525: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 180 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9.1 KiB/s rd, 4 op/s; 879/8511 objects degraded (10.328%); 207/8511 objects misplaced (2.432%) 2024-08-17T17:48:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:50.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:50 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[104516]: pgmap v1525: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 180 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9.1 KiB/s rd, 4 op/s; 879/8511 objects degraded (10.328%); 207/8511 objects misplaced (2.432%) 2024-08-17T17:48:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:50.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[109246]: pgmap v1525: 119 pgs: 1 active+clean+remapped, 23 active+undersized, 20 active+undersized+degraded, 75 active+clean; 180 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9.1 KiB/s rd, 4 op/s; 879/8511 objects degraded (10.328%); 207/8511 objects misplaced (2.432%) 2024-08-17T17:48:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:50 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:51.000 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:51.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.022 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:51.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:51.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:51.147 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:51.235 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:48:51.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:51.301 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:51.402 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.424 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.454 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:51.526 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:51.548 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:51.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:51.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:51.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:51.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:51.802 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:51.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:51.931 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:52.034 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:52.058 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:52.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:52.187 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:52.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:52.315 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:52.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.445 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:52.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:52.546 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:52.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.572 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:52.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:52.673 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:52.697 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:52.724 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-08-17T17:48:52.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-08-17T17:48:52.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:52.823 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:52.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:52.847 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:52 smithi175 ceph-mon[101434]: pgmap v1526: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 24 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 99/1023 objects degraded (9.677%) 2024-08-17T17:48:52.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:52.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:52 smithi064 ceph-mon[104516]: pgmap v1526: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 24 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 99/1023 objects degraded (9.677%) 2024-08-17T17:48:52.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:52 smithi064 ceph-mon[109246]: pgmap v1526: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 24 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 99/1023 objects degraded (9.677%) 2024-08-17T17:48:52.947 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:53.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:53.070 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:53.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:53.172 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:53.233 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:53.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:53.360 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:53.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:53.486 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:53.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:53.611 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:53.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:53.738 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:53.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:53.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:53 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 99/1023 objects degraded (9.677%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:53.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:53 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 99/1023 objects degraded (9.677%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:53.863 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:53.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:53 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 99/1023 objects degraded (9.677%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:53.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:54.000 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:54.025 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:54.029 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:54.031 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:54.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:54.125 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:48:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:48:53] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-08-17T17:48:54.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:54.251 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:54.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.375 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:54.477 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:54.503 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:54.545 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2024-08-17T17:48:54.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.627 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:54.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:54.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.838 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:54 smithi175 ceph-mon[101434]: pgmap v1527: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 12 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 50/456 objects degraded (10.965%) 2024-08-17T17:48:54.838 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2088219308' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.839 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:54.874 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:54.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[104516]: pgmap v1527: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 12 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 50/456 objects degraded (10.965%) 2024-08-17T17:48:54.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2088219308' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.897 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[109246]: pgmap v1527: 119 pgs: 1 active+clean+remapped, 24 active+undersized, 19 active+undersized+degraded, 75 active+clean; 12 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 50/456 objects degraded (10.965%) 2024-08-17T17:48:54.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2088219308' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:54.977 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:55.000 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:55.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:55.126 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:55.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:55.250 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:55.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:55.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:55.374 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:55.476 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:55.499 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:55.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:55.509 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:89166 10.d 10:b7530a99:::benchmark_data_smithi064_306376_object89165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:45:47.329345+0000", 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 188.178049985, 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.29209126600000002, 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.510 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 89166 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.329345+0000", 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.329345+0000", 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.511 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.329344+0000", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.329721+0000", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00037614799999999998 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.412955+0000", 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.083234553000000003 2024-08-17T17:48:55.512 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.412974+0000", 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8709999999999999e-05 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.413226+0000", 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025267900000000001 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.513 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.413410+0000", 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000183358 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.413548+0000", 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013834700000000001 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.423709+0000", 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.010161125 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.514 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.423740+0000", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.0548e-05 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.621355+0000", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.19761546499999999 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.621398+0000", 2024-08-17T17:48:55.515 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2750000000000002e-05 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:45:47.621437+0000", 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8602000000000001e-05 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:151795 10.3 e436/413 10:cf3d28b6:::benchmark_data_smithi064_306376_object26814:head v 436'9492, mlcod=436'9490)", 2024-08-17T17:48:55.516 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.681927+0000", 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.825468331, 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28891272600000001, 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681927+0000", 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.517 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681927+0000", 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681926+0000", 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681933+0000", 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9619999999999996e-06 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.518 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681954+0000", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.0324999999999999e-05 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.681960+0000", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6640000000000001e-06 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.682026+0000", 2024-08-17T17:48:55.519 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5667000000000007e-05 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.682045+0000", 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8649999999999999e-05 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.970840+0000", 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28879508100000001 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.520 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:151802 10.3 e436/413 10:cb119f89:::benchmark_data_smithi064_306376_object26821:head v 436'9493, mlcod=436'9490)", 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.686557+0000", 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.82083881, 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28429059400000001, 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686557+0000", 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.521 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686557+0000", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686555+0000", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686563+0000", 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9379999999999992e-06 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.522 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686579+0000", 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5787999999999999e-05 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.686586+0000", 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3329999999999999e-06 2024-08-17T17:48:55.523 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687235+0000", 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00064828300000000004 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.524 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687326+0000", 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.1304000000000005e-05 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.970847+0000", 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28352116300000002 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.525 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:151801 10.1 10:8f471a43:::benchmark_data_smithi064_306376_object26820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.687731+0000", 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.819664385, 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.32208416899999998, 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.526 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151801 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687731+0000", 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687731+0000", 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.527 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687730+0000", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687744+0000", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3546999999999999e-05 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687751+0000", 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1450000000000002e-06 2024-08-17T17:48:55.528 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687756+0000", 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687864+0000", 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000107555 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.529 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.687987+0000", 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012340800000000001 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.688114+0000", 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012692199999999999 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.691835+0000", 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0037205760000000002 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.530 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.699708+0000", 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0078734040000000005 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.009799+0000", 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31009104900000001 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.531 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.009815+0000", 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6110999999999999e-05 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:151807 10.2 e436/410 10:41f7a2db:::benchmark_data_smithi064_306376_object26826:head v 436'9467, mlcod=436'9465)", 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.694329+0000", 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.813066324, 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31632866999999998, 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.532 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694329+0000", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694329+0000", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694327+0000", 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.533 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694339+0000", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1409999999999999e-05 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694360+0000", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1254e-05 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694370+0000", 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8849999999999994e-06 2024-08-17T17:48:55.534 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694510+0000", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014061299999999999 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.694533+0000", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2302000000000001e-05 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.010658+0000", 2024-08-17T17:48:55.535 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.316124972 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:151808 10.c e437/410 10:3a9117c9:::benchmark_data_smithi064_306376_object26827:head v 437'9533, mlcod=436'9531)", 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.698654+0000", 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.808741814, 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.37355071499999998, 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.536 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698654+0000", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698654+0000", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698653+0000", 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.537 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698660+0000", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6259999999999997e-06 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698670+0000", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0570999999999999e-05 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.698677+0000", 2024-08-17T17:48:55.538 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.4450000000000001e-06 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.700788+0000", 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021112539999999999 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.700810+0000", 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.1841999999999999e-05 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.539 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.071978+0000", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.371167725 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.072204+0000", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00022654899999999999 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:151801 10.11 10:8f471a43:::benchmark_data_smithi064_306376_object26820:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.702663+0000", 2024-08-17T17:48:55.540 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.80473214600001, 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.04667369, 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151801 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702663+0000", 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.541 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702663+0000", 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702661+0000", 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:48:55.542 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702670+0000", 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.3010000000000007e-06 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702682+0000", 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2476999999999999e-05 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.543 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.702690+0000", 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1070000000000005e-06 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.010723+0000", 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.30803247299999997 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.010733+0000", 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8500000000000006e-06 2024-08-17T17:48:55.544 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.749221+0000", 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.73848880299999997 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.749337+0000", 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000115578 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.545 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_repop(client.55666.0:151804 10.0 e436/410)", 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.715187+0000", 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.79220870100001, 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28447382199999999, 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715187+0000", 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.546 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715187+0000", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715184+0000", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715199+0000", 2024-08-17T17:48:55.547 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4707e-05 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715217+0000", 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.789e-05 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.715228+0000", 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1440999999999999e-05 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.548 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.999605+0000", 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.28437623699999998 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.999660+0000", 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5760999999999997e-05 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:151810 10.e 10:7aad0fe4:::benchmark_data_smithi064_306376_object26829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:48:55.549 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:46:51.718131+0000", 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 123.78926434100001, 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.31956926299999999, 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 151810 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.550 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718131+0000", 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718131+0000", 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718129+0000", 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.551 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718141+0000", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1557e-05 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718160+0000", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.9499000000000001e-05 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718171+0000", 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1034e-05 2024-08-17T17:48:55.552 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718217+0000", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6288999999999997e-05 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718348+0000", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013063800000000001 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.718464+0000", 2024-08-17T17:48:55.553 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011597800000000001 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.734246+0000", 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.015781573 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:51.759458+0000", 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.025212801999999999 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.554 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.037551+0000", 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27809233100000003 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.037656+0000", 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104769 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:46:52.037700+0000", 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4789000000000001e-05 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.555 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228954 10.e 10:7611a06c:::benchmark_data_smithi064_306376_object103973:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.885856+0000", 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.621539608999999, 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.71786666200000004, 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228954 2024-08-17T17:48:55.556 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885856+0000", 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885856+0000", 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.557 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885854+0000", 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885866+0000", 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2007999999999999e-05 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885879+0000", 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.375e-05 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.558 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885888+0000", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.4400000000000005e-06 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887878+0000", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0019898910000000001 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op must wait for map", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887884+0000", 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5819999999999998e-06 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.559 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.068684+0000", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18079914999999999 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.068694+0000", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0101999999999999e-05 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.603654+0000", 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.53495989099999997 2024-08-17T17:48:55.560 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.603722+0000", 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8918000000000007e-05 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228960 10.1e 10:7d34d26a:::benchmark_data_smithi064_306376_object103979:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.885904+0000", 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.621491571, 2024-08-17T17:48:55.561 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.74929308100000003, 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228960 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885904+0000", 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.562 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885904+0000", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885903+0000", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885906+0000", 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6989999999999999e-06 2024-08-17T17:48:55.563 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885912+0000", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9819999999999996e-06 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885917+0000", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5959999999999997e-06 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.068984+0000", 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18306747800000001 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.564 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.068991+0000", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.6649999999999997e-06 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.632388+0000", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56339720000000004 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.632507+0000", 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011897500000000001 2024-08-17T17:48:55.565 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.635122+0000", 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0026148059999999999 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.635197+0000", 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5080000000000006e-05 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.566 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228961 10.d 10:b175d3d8:::benchmark_data_smithi064_306376_object103980:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.885923+0000", 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.621472370999999, 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.71226449000000003, 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228961 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.567 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885923+0000", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885923+0000", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885923+0000", 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.568 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885925+0000", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.553e-06 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885930+0000", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.5669999999999996e-06 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.885934+0000", 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8979999999999998e-06 2024-08-17T17:48:55.569 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.141043+0000", 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25510974600000003 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.141053+0000", 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6749999999999997e-06 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.570 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.595235+0000", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45418149800000002 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.595369+0000", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013464999999999999 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.598099+0000", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0027297490000000001 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.571 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.598187+0000", 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.8471e-05 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228962 10.1e 10:7bbda1a1:::benchmark_data_smithi064_306376_object103981:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.887866+0000", 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.619529076999999, 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.75073885500000004, 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.572 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228962 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887866+0000", 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887866+0000", 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.573 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887864+0000", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887875+0000", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0505e-05 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887886+0000", 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1661000000000001e-05 2024-08-17T17:48:55.574 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887896+0000", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.6940000000000007e-06 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.069007+0000", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.18111126599999999 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.069012+0000", 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2089999999999996e-06 2024-08-17T17:48:55.575 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.634658+0000", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.565645183 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.634783+0000", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012484 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.638544+0000", 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.003761936 2024-08-17T17:48:55.576 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.638605+0000", 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.0695000000000001e-05 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228963 10.21 10:8749e864:::benchmark_data_smithi064_306376_object103982:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.887901+0000", 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.619494777, 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.70856711800000005, 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.577 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228963 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887901+0000", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.578 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887901+0000", 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887900+0000", 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887903+0000", 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.1999999999999999e-06 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.579 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887909+0000", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2660000000000001e-06 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.887914+0000", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3190000000000002e-06 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.147719+0000", 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.25980510400000001 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.580 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.581 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.581 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.147727+0000", 2024-08-17T17:48:55.581 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.8199999999999997e-06 2024-08-17T17:48:55.581 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.594131+0000", 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.44640412800000001 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for missing object", 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.594292+0000", 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016108999999999999 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.582 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.596412+0000", 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021199159999999999 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.596468+0000", 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5628999999999999e-05 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.583 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228972 10.10 10:0fe822cd:::benchmark_data_smithi064_306376_object103991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:27.921872+0000", 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.585523492, 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.27723821999999998, 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228972 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.584 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921872+0000", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921872+0000", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921869+0000", 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999967 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.585 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921885+0000", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6104999999999999e-05 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921918+0000", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2668e-05 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.921933+0000", 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5254e-05 2024-08-17T17:48:55.586 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.927911+0000", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0059783049999999997 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.928064+0000", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000153258 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.928608+0000", 2024-08-17T17:48:55.587 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00054382300000000003 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:27.961203+0000", 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.032595115000000001 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.193832+0000", 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.23262844699999999 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.588 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.199028+0000", 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0051963340000000004 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.199072+0000", 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3572999999999998e-05 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.199110+0000", 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.8485999999999997e-05 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.589 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:228974 10.d 10:b454d9ab:::benchmark_data_smithi064_306376_object103993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:28.044113+0000", 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 27.463282654, 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.59234290099999998, 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 228974 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.590 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044113+0000", 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044113+0000", 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.591 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044110+0000", 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999976 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044121+0000", 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.0740999999999999e-05 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044146+0000", 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4566000000000002e-05 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.592 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.044157+0000", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1232e-05 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.141103+0000", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.096946141999999999 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.141113+0000", 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4469999999999995e-06 2024-08-17T17:48:55.593 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.595658+0000", 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.45454545800000001 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.595787+0000", 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012927699999999999 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:48:55.594 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.596153+0000", 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00036536700000000001 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.599400+0000", 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0032472030000000002 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.600013+0000", 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00061342299999999998 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.595 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.636333+0000", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.036319702000000002 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.636376+0000", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3167999999999997e-05 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:28.636456+0000", 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9504000000000003e-05 2024-08-17T17:48:55.596 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:238368 10.1c 10:3dc5a165:::benchmark_data_smithi064_306376_object113387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:38.002898+0000", 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.504496988, 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56371603299999995, 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.597 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 238368 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002898+0000", 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002898+0000", 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.598 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002897+0000", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002903+0000", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002912+0000", 2024-08-17T17:48:55.599 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.8209999999999997e-06 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002920+0000", 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.5530000000000004e-06 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002971+0000", 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0704000000000001e-05 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.600 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.002981+0000", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.039e-05 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.561824+0000", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.55884303800000001 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.561973+0000", 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014867099999999999 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.601 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.562325+0000", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00035224099999999999 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.564475+0000", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021498709999999998 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.566471+0000", 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0019965510000000001 2024-08-17T17:48:55.602 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.566519+0000", 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7636999999999998e-05 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.566614+0000", 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.5302000000000003e-05 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:238371 10.1c 10:38b8b82d:::benchmark_data_smithi064_306376_object113390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:38.003281+0000", 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.504114085000001, 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56421258600000002, 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 238371 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.604 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003281+0000", 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003281+0000", 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.605 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003281+0000", 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003285+0000", 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.814e-06 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003295+0000", 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7669999999999998e-06 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.606 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003302+0000", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0740000000000004e-06 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003351+0000", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9206000000000003e-05 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.003359+0000", 2024-08-17T17:48:55.607 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.6000000000000001e-06 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.564546+0000", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.56118650000000003 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.564685+0000", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00013966400000000001 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.565019+0000", 2024-08-17T17:48:55.608 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00033398 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.566645+0000", 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0016253890000000001 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.567377+0000", 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00073234899999999998 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.609 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.567419+0000", 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2240999999999998e-05 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.567494+0000", 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.4720000000000003e-05 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:238379 10.1 10:80f99eee:::benchmark_data_smithi064_306376_object113398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:48:55.610 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:38.014910+0000", 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.492485468000002, 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.52468607300000003, 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 238379 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.611 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014910+0000", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014910+0000", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014909+0000", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.612 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014914+0000", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9089999999999999e-06 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014926+0000", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2541000000000001e-05 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014934+0000", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7600000000000002e-06 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.613 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014977+0000", 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.2657999999999997e-05 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.014985+0000", 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.0299999999999994e-06 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.536904+0000", 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.52191928499999996 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.614 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.537041+0000", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000137346 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 0", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.537380+0000", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00033890800000000002 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.539064+0000", 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0016835800000000001 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.615 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.539455+0000", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00039071800000000002 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.539514+0000", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.9586000000000002e-05 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.539596+0000", 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1705000000000005e-05 2024-08-17T17:48:55.616 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "osd_op(client.55666.0:238385 10.10 10:0ea73243:::benchmark_data_smithi064_306376_object113404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:48:38.021415+0000", 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 17.485980356999999, 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.54047713799999997, 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "client_info": { 2024-08-17T17:48:55.617 INFO:teuthology.orchestra.run.smithi175.stdout: "client": "client.55666", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "client_addr": "172.21.15.64:0/2115630191", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "tid": 238385 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021415+0000", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021415+0000", 2024-08-17T17:48:55.618 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021413+0000", 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021421+0000", 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:48:55.619 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021435+0000", 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.4549000000000001e-05 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021443+0000", 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.644e-06 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021645+0000", 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00020167000000000001 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.620 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for peered", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.021652+0000", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9049999999999997e-06 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.555707+0000", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.534054959 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.555845+0000", 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000137236 2024-08-17T17:48:55.621 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "waiting for subops from 6", 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.556473+0000", 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00062831300000000003 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "op_commit", 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.559548+0000", 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0030752280000000002 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:55.622 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.561743+0000", 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0021951140000000002 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "commit_sent", 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.561797+0000", 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.414e-05 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:48:38.561892+0000", 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.4745000000000002e-05 2024-08-17T17:48:55.623 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.624 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.624 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.624 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:48:55.624 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:48:55.624 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:48:55.624 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:55.645 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: osdmap e491: 8 total, 7 up, 8 in 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:55.747 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:55.768 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:55.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:55.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:55.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: osdmap e491: 8 total, 7 up, 8 in 2024-08-17T17:48:55.877 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: osdmap e491: 8 total, 7 up, 8 in 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:55.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:55.892 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:55.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:56.016 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:56.117 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:56.138 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:56.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:56.263 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:56.365 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:56.387 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:56.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:48:56.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:56.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:56.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:56.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.635 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.686 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:56.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:56.759 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:56.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:56.882 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:56.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:57.006 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[104516]: pgmap v1529: 119 pgs: 1 active+clean+remapped, 25 active+undersized, 18 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[109246]: pgmap v1529: 119 pgs: 1 active+clean+remapped, 25 active+undersized, 18 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:57.131 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:57.137 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:56 smithi175 ceph-mon[101434]: pgmap v1529: 119 pgs: 1 active+clean+remapped, 25 active+undersized, 18 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:57.138 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.138 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:48:57.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:57.358 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:57.379 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:57.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:57.504 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:57.605 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:57.627 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:57.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:57.751 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:57.853 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:57.875 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:57.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:57.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:57.999 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:58.101 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:58.124 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:58.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:58.248 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:58.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:58.353 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:48:58.353 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:48:58.353 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:48:58.354 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:48:58.377 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:58.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:58.505 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:58.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:58.629 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:58.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:58.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.754 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:58.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:58.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:48:58.855 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:58.877 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:58 smithi064 ceph-mon[104516]: pgmap v1530: 119 pgs: 1 active+clean+remapped, 30 active+undersized, 13 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:58.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:48:58 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:58.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:58 smithi064 ceph-mon[109246]: pgmap v1530: 119 pgs: 1 active+clean+remapped, 30 active+undersized, 13 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:58.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:48:58 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:58.979 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:59.001 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:59.103 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:59.127 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:59.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:58 smithi175 ceph-mon[101434]: pgmap v1530: 119 pgs: 1 active+clean+remapped, 30 active+undersized, 13 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:48:59.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:48:58 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:59.216 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:48:59.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.380 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.402 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.462 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:59.503 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.525 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:59.651 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:59.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.775 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:48:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.876 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:59.898 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:48:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:00.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:00.023 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:00.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:00.125 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:00.147 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:00.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.269 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.371 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:00.392 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:00.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:00.520 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:00.623 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:00.643 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:00.745 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:00.767 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:00.869 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:00 smithi064 ceph-mon[104516]: pgmap v1531: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:00.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1327888905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:00 smithi064 ceph-mon[109246]: pgmap v1531: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:00.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1327888905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:00.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:00.999 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:00 smithi175 ceph-mon[101434]: pgmap v1531: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:00.999 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1327888905' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:01.015 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:01.016 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:01.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:01.119 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.142 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:01.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.237 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:01.237 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:01.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:01.265 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:01.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:01.389 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:01.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:01.512 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:01.614 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:01.636 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:01.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:01.760 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:01.861 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:01.868 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' removed 2024-08-17T17:49:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.891 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:01.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:01.923 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:49:01.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.019 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:02.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:02.115 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:01 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:49:02.116 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:01 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:49:02.121 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:02.143 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:01 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:49:02.203 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:02.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:02.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.268 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:02.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:02.369 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.391 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:02.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:02.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:49:02.151+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e492 definitely_dead 0 2024-08-17T17:49:02.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:02.515 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:02.615 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:49:02.615 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-17T17:49:02.616 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T17:49:02.616 INFO:tasks.workunit:Pulling workunits from ref reef 2024-08-17T17:49:02.616 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T17:49:02.617 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:49:02.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.633 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T17:49:02.633 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:49:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.640 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:02.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:02.684 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T17:49:02.684 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T17:49:02.739 INFO:tasks.workunit:timeout=3h 2024-08-17T17:49:02.739 INFO:tasks.workunit:cleanup=True 2024-08-17T17:49:02.740 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:49:02.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:02.802 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:02.833 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T17:49:02.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: pgmap v1532: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:02.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:49:02.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: osdmap e492: 8 total, 7 up, 8 in 2024-08-17T17:49:02.880 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1348775995' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:02.880 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:02.880 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[104516]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T17:49:02.880 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.6. 2024-08-17T17:49:02.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: pgmap v1532: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:02.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:49:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: osdmap e492: 8 total, 7 up, 8 in 2024-08-17T17:49:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1348775995' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:02.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:02 smithi175 ceph-mon[101434]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T17:49:02.925 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.027 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.149 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:03.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: pgmap v1532: 119 pgs: 1 active+clean+remapped, 31 active+undersized, 12 active+undersized+degraded, 75 active+clean; 8.2 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 34/258 objects degraded (13.178%) 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/10719922' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: osdmap e492: 8 total, 7 up, 8 in 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1348775995' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T17:49:03.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:02 smithi064 ceph-mon[109246]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T17:49:03.272 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:03.294 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:03.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:03.418 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:03.519 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.546 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.648 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:03.670 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:03.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:03.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:03.838 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:03.838 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:03.896 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.917 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:03.917 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:03.918 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:04.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:04.043 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:04.144 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:04.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:03 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T17:49:04.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:03 smithi175 ceph-mon[101434]: osdmap e493: 8 total, 7 up, 7 in 2024-08-17T17:49:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:03 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:04.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:03 smithi175 ceph-mon[101434]: osdmap e494: 8 total, 7 up, 7 in 2024-08-17T17:49:04.165 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:04.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:03] "GET /metrics HTTP/1.1" 200 36706 "" "Prometheus/2.51.0" 2024-08-17T17:49:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T17:49:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[104516]: osdmap e493: 8 total, 7 up, 7 in 2024-08-17T17:49:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:04.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[104516]: osdmap e494: 8 total, 7 up, 7 in 2024-08-17T17:49:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T17:49:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[109246]: osdmap e493: 8 total, 7 up, 7 in 2024-08-17T17:49:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 34/258 objects degraded (13.178%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:04.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:03 smithi064 ceph-mon[109246]: osdmap e494: 8 total, 7 up, 7 in 2024-08-17T17:49:04.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:04.290 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:04.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.417 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:04.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:04.498 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:49:04.518 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:04.562 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:04.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:04.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:04.710 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:04.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:04.810 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:04.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:04.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:05.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:05.059 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:05.146 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:04 smithi175 ceph-mon[101434]: pgmap v1535: 83 pgs: 23 active+undersized, 11 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:05.147 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:04 smithi175 ceph-mon[101434]: osdmap e495: 8 total, 7 up, 7 in 2024-08-17T17:49:05.160 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:05.182 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:05.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:04 smithi064 ceph-mon[109246]: pgmap v1535: 83 pgs: 23 active+undersized, 11 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:05.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:04 smithi064 ceph-mon[109246]: osdmap e495: 8 total, 7 up, 7 in 2024-08-17T17:49:05.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:04 smithi064 ceph-mon[104516]: pgmap v1535: 83 pgs: 23 active+undersized, 11 active+undersized+degraded, 49 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:05.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:04 smithi064 ceph-mon[104516]: osdmap e495: 8 total, 7 up, 7 in 2024-08-17T17:49:05.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:05.307 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:05.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:05.431 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:05.454 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:05.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:05.556 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:05.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:05.680 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:05.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:05.804 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:05.905 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:05.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:05.926 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:06.052 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:06.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:06.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:06.176 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:06.278 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:06.299 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:06.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.303 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:06.303 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:06.401 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:06.423 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:06.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:06.550 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:06.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:06.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:06.674 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:06.776 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:06.798 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:06 smithi064 ceph-mon[104516]: pgmap v1538: 83 pgs: 1 unknown, 9 peering, 20 active+undersized, 11 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:06.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:06 smithi064 ceph-mon[104516]: osdmap e496: 8 total, 7 up, 7 in 2024-08-17T17:49:06.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:06 smithi064 ceph-mon[109246]: pgmap v1538: 83 pgs: 1 unknown, 9 peering, 20 active+undersized, 11 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:06.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:06 smithi064 ceph-mon[109246]: osdmap e496: 8 total, 7 up, 7 in 2024-08-17T17:49:06.900 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:06.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:06 smithi175 ceph-mon[101434]: pgmap v1538: 83 pgs: 1 unknown, 9 peering, 20 active+undersized, 11 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2024-08-17T17:49:06.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:06 smithi175 ceph-mon[101434]: osdmap e496: 8 total, 7 up, 7 in 2024-08-17T17:49:06.921 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.944 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:49:07.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:07.047 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:07.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:07.171 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:07.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:07.187 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:07.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:07.305 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:07.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:07.439 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:07.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:07.562 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:07.584 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:07 smithi175 ceph-mon[101434]: osdmap e497: 8 total, 7 up, 7 in 2024-08-17T17:49:07.585 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:07 smithi175 ceph-mon[101434]: 1.0 scrub starts 2024-08-17T17:49:07.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:07.686 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:07.788 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.811 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:07.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:07.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:07 smithi064 ceph-mon[104516]: osdmap e497: 8 total, 7 up, 7 in 2024-08-17T17:49:07.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:07 smithi064 ceph-mon[104516]: 1.0 scrub starts 2024-08-17T17:49:07.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:07 smithi064 ceph-mon[109246]: osdmap e497: 8 total, 7 up, 7 in 2024-08-17T17:49:07.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:07 smithi064 ceph-mon[109246]: 1.0 scrub starts 2024-08-17T17:49:07.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:07.932 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:08.034 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:08.055 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:08.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.180 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:08.217 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:08.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:08.406 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:08.428 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:08.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:08.529 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:08.537 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:49:08.551 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:08.590 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:08 smithi175 ceph-mon[101434]: pgmap v1541: 83 pgs: 6 active+clean+remapped, 1 unknown, 12 peering, 16 active+undersized, 9 active+undersized+degraded, 39 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.2 KiB/s, 3 objects/s recovering 2024-08-17T17:49:08.590 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:08 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:08.590 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:08 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[104516]: pgmap v1541: 83 pgs: 6 active+clean+remapped, 1 unknown, 12 peering, 16 active+undersized, 9 active+undersized+degraded, 39 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.2 KiB/s, 3 objects/s recovering 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[109246]: pgmap v1541: 83 pgs: 6 active+clean+remapped, 1 unknown, 12 peering, 16 active+undersized, 9 active+undersized+degraded, 39 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.2 KiB/s, 3 objects/s recovering 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:08.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:08 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 11/84 objects degraded (13.095%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:08.653 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.677 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:08.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:08.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.801 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:08.815 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:08.815 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:08.823 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T17:49:08.879 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T17:49:08.881 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:49:08.881 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T17:49:08.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:08.924 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:08.940 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T17:49:08.991 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T17:49:09.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:09.038 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:49:09.040 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:49:09.040 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T17:49:09.048 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:09.087 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:49:09.091 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2024-08-17T17:49:09.091 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T17:49:09.150 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:09.157 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-08-17T17:49:09.158 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-08-17T17:49:09.158 DEBUG:teuthology.orchestra.run.smithi064:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-08-17T17:49:09.170 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:09.273 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:09.294 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:09.391 INFO:tasks.workunit.client.0.smithi064.stdout:Running main() from gmock_main.cc 2024-08-17T17:49:09.391 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] Running 70 tests from 1 test suite. 2024-08-17T17:49:09.391 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment set-up. 2024-08-17T17:49:09.391 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd 2024-08-17T17:49:09.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:09.420 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:09.517 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:49:09.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:09.544 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:09.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:09.646 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:09.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: osdmap e498: 8 total, 7 up, 7 in 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:09.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:09 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.671 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:09.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: osdmap e498: 8 total, 7 up, 7 in 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1751094249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: osdmap e498: 8 total, 7 up, 7 in 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:09.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:09.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:09 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:09.720 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:09.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:09.795 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:09.897 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:09.920 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:10.044 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:10.095 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 7, 4, 5, 3] out_osds: [6] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:49:10.095 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:10.096 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 7, 4, 5, 3] 2024-08-17T17:49:10.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd out 1 2024-08-17T17:49:10.146 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.167 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:10.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:10.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.292 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:10.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:10.393 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.397 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:10.414 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.516 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: pgmap v1543: 83 pgs: 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 3 peering, 15 active+undersized, 9 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 KiB/s rd, 40 KiB/s wr, 105 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.0 KiB/s, 8 objects/s recovering 2024-08-17T17:49:10.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:49:10.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: osdmap e499: 8 total, 7 up, 7 in 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: pgmap v1543: 83 pgs: 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 3 peering, 15 active+undersized, 9 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 KiB/s rd, 40 KiB/s wr, 105 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.0 KiB/s, 8 objects/s recovering 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: osdmap e499: 8 total, 7 up, 7 in 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:10.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:10.639 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:10.660 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:10.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:10.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.785 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:10.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:10.839 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: pgmap v1543: 83 pgs: 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 3 peering, 15 active+undersized, 9 active+undersized+degraded, 46 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 KiB/s rd, 40 KiB/s wr, 105 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 1.0 KiB/s, 8 objects/s recovering 2024-08-17T17:49:10.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:49:10.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: osdmap e499: 8 total, 7 up, 7 in 2024-08-17T17:49:10.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:49:10.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:10.840 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:10.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:10.908 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:11.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:11.114 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_all_features 2024-08-17T17:49:11.132 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:11.154 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:11.178 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_all_features (64 ms) 2024-08-17T17:49:11.179 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.copyup 2024-08-17T17:49:11.256 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:11.278 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:11.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:11.347 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.copyup (169 ms) 2024-08-17T17:49:11.348 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-08-17T17:49:11.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparse_copyup (17 ms) 2024-08-17T17:49:11.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-08-17T17:49:11.364 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:11.365 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:11.371 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-08-17T17:49:11.371 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-08-17T17:49:11.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:11.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-08-17T17:49:11.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.directory_methods 2024-08-17T17:49:11.403 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:11.410 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.directory_methods (29 ms) 2024-08-17T17:49:11.410 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.create 2024-08-17T17:49:11.438 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.create (28 ms) 2024-08-17T17:49:11.438 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_features 2024-08-17T17:49:11.439 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-08-17T17:49:11.440 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-08-17T17:49:11.442 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-08-17T17:49:11.442 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-08-17T17:49:11.443 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2024-08-17T17:49:11.443 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-08-17T17:49:11.445 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-08-17T17:49:11.445 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-08-17T17:49:11.446 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-08-17T17:49:11.446 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-08-17T17:49:11.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-08-17T17:49:11.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_size 2024-08-17T17:49:11.453 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-08-17T17:49:11.453 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_size 2024-08-17T17:49:11.457 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-08-17T17:49:11.458 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.protection_status 2024-08-17T17:49:11.475 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2024-08-17T17:49:11.475 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-08-17T17:49:11.482 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-08-17T17:49:11.483 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v1 2024-08-17T17:49:11.505 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:11.524 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2024-08-17T17:49:11.524 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v2 2024-08-17T17:49:11.527 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:11.555 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2024-08-17T17:49:11.555 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshots 2024-08-17T17:49:11.575 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2024-08-17T17:49:11.575 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapid_race 2024-08-17T17:49:11.579 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2024-08-17T17:49:11.579 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.stripingv2 2024-08-17T17:49:11.589 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2024-08-17T17:49:11.589 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_save 2024-08-17T17:49:11.593 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-08-17T17:49:11.593 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-08-17T17:49:11.599 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2024-08-17T17:49:11.599 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_update 2024-08-17T17:49:11.605 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2024-08-17T17:49:11.605 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-08-17T17:49:11.605 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-08-17T17:49:11.605 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-08-17T17:49:11.610 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_add (5 ms) 2024-08-17T17:49:11.610 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-08-17T17:49:11.615 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2024-08-17T17:49:11.615 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.flags 2024-08-17T17:49:11.620 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-08-17T17:49:11.620 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.metadata 2024-08-17T17:49:11.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:11.645 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.metadata (25 ms) 2024-08-17T17:49:11.645 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_features 2024-08-17T17:49:11.650 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:11.652 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-08-17T17:49:11.652 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror 2024-08-17T17:49:11.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror (38 ms) 2024-08-17T17:49:11.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image 2024-08-17T17:49:11.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:49:11.700 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:11.707 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image (18 ms) 2024-08-17T17:49:11.707 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-08-17T17:49:11.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:49:11.724 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:11.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:49:11.745 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:11.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:11.768 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:49:11.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:11.779 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:11.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:49:11.791 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:11.812 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:49:11.812 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:11.835 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:49:11.835 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:11.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:11.907 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:12.009 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:12.030 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:12.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:49:12.133 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:12.154 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:12.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:12.284 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:12.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:12 smithi175 ceph-mon[101434]: pgmap v1545: 115 pgs: 9 creating+peering, 23 unknown, 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 1 peering, 15 active+undersized, 9 active+undersized+degraded, 48 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 122 KiB/s rd, 33 KiB/s wr, 86 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 866 B/s, 7 objects/s recovering 2024-08-17T17:49:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:49:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:12 smithi175 ceph-mon[101434]: osdmap e500: 8 total, 7 up, 7 in 2024-08-17T17:49:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:12 smithi175 ceph-mon[101434]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:49:12.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:12 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3824867967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:12.413 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:12.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[109246]: pgmap v1545: 115 pgs: 9 creating+peering, 23 unknown, 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 1 peering, 15 active+undersized, 9 active+undersized+degraded, 48 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 122 KiB/s rd, 33 KiB/s wr, 86 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 866 B/s, 7 objects/s recovering 2024-08-17T17:49:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:49:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[109246]: osdmap e500: 8 total, 7 up, 7 in 2024-08-17T17:49:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[109246]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:49:12.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3824867967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:12.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[104516]: pgmap v1545: 115 pgs: 9 creating+peering, 23 unknown, 9 active+clean+remapped, 1 active+clean+remapped+scrubbing, 1 peering, 15 active+undersized, 9 active+undersized+degraded, 48 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 122 KiB/s rd, 33 KiB/s wr, 86 op/s; 11/84 objects degraded (13.095%); 5/84 objects misplaced (5.952%); 866 B/s, 7 objects/s recovering 2024-08-17T17:49:12.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1075764332' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-336945-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:49:12.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[104516]: osdmap e500: 8 total, 7 up, 7 in 2024-08-17T17:49:12.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[104516]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:49:12.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:12 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3824867967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:12.515 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:12.537 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:12.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:12.665 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:12.767 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:12.789 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:12.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:12.917 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:13.023 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:13.046 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:13.048 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:13.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.174 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:13.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:13.296 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:13 smithi175 ceph-mon[101434]: osdmap e501: 8 total, 7 up, 7 in 2024-08-17T17:49:13.297 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:13.424 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:13 smithi064 ceph-mon[104516]: osdmap e501: 8 total, 7 up, 7 in 2024-08-17T17:49:13.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:13 smithi064 ceph-mon[109246]: osdmap e501: 8 total, 7 up, 7 in 2024-08-17T17:49:13.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:49:13.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:13.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:13.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:13.682 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:13.753 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:13.784 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.807 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:13.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.934 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:14.057 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:14.149 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:13] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-08-17T17:49:14.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[104516]: pgmap v1548: 115 pgs: 9 creating+peering, 15 unknown, 10 active+clean+remapped, 18 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 505 KiB/s rd, 348 KiB/s wr, 869 op/s; 12/96 objects degraded (12.500%); 5/96 objects misplaced (5.208%) 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[104516]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[104516]: 1.0 scrub starts 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[104516]: 1.0 scrub ok 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 12/96 objects degraded (12.500%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[109246]: pgmap v1548: 115 pgs: 9 creating+peering, 15 unknown, 10 active+clean+remapped, 18 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 505 KiB/s rd, 348 KiB/s wr, 869 op/s; 12/96 objects degraded (12.500%); 5/96 objects misplaced (5.208%) 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[109246]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[109246]: 1.0 scrub starts 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[109246]: 1.0 scrub ok 2024-08-17T17:49:14.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:14 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 12/96 objects degraded (12.500%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:14.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:14.184 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:14.276 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_status (2569 ms) 2024-08-17T17:49:14.276 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-08-17T17:49:14.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:14.308 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:14.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_map (88 ms) 2024-08-17T17:49:14.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-08-17T17:49:14.371 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-08-17T17:49:14.371 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-08-17T17:49:14.389 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_snapshot (17 ms) 2024-08-17T17:49:14.389 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-08-17T17:49:14.391 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-08-17T17:49:14.391 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-08-17T17:49:14.392 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-08-17T17:49:14.393 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-08-17T17:49:14.395 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-08-17T17:49:14.395 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-08-17T17:49:14.401 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-08-17T17:49:14.401 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-08-17T17:49:14.404 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2024-08-17T17:49:14.405 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-08-17T17:49:14.408 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-08-17T17:49:14.409 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_add 2024-08-17T17:49:14.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:14 smithi175 ceph-mon[101434]: pgmap v1548: 115 pgs: 9 creating+peering, 15 unknown, 10 active+clean+remapped, 18 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 505 KiB/s rd, 348 KiB/s wr, 869 op/s; 12/96 objects degraded (12.500%); 5/96 objects misplaced (5.208%) 2024-08-17T17:49:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:14 smithi175 ceph-mon[101434]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:49:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:14 smithi175 ceph-mon[101434]: 1.0 scrub starts 2024-08-17T17:49:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:14 smithi175 ceph-mon[101434]: 1.0 scrub ok 2024-08-17T17:49:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:14 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 12/96 objects degraded (12.500%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:14.412 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-08-17T17:49:14.412 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-08-17T17:49:14.414 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2024-08-17T17:49:14.415 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_list 2024-08-17T17:49:14.417 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2024-08-17T17:49:14.417 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-08-17T17:49:14.423 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2024-08-17T17:49:14.423 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_add 2024-08-17T17:49:14.426 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-08-17T17:49:14.426 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-08-17T17:49:14.430 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2024-08-17T17:49:14.430 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_get 2024-08-17T17:49:14.432 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-08-17T17:49:14.432 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-08-17T17:49:14.434 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-08-17T17:49:14.434 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-08-17T17:49:14.436 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-08-17T17:49:14.436 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-08-17T17:49:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:14.440 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2024-08-17T17:49:14.440 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-08-17T17:49:14.443 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-08-17T17:49:14.443 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-719-gced649f8/rpm/el9/BUILD/ceph-18.2.4-719-gced649f8/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout:Expected equality of these values: 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout: 1U 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout: Which is: 1 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout: keys.size() 2024-08-17T17:49:14.446 INFO:tasks.workunit.client.0.smithi064.stdout: Which is: 3 2024-08-17T17:49:14.447 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] TestClsRbd.group_snap_set (3 ms) 2024-08-17T17:49:14.447 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-08-17T17:49:14.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-08-17T17:49:14.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-08-17T17:49:14.465 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-08-17T17:49:14.465 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-08-17T17:49:14.540 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:14.561 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:14.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:14.667 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (203 ms) 2024-08-17T17:49:14.668 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-08-17T17:49:14.671 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-08-17T17:49:14.671 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-08-17T17:49:14.673 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2024-08-17T17:49:14.673 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.trash_methods 2024-08-17T17:49:14.681 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-08-17T17:49:14.681 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.op_features 2024-08-17T17:49:14.687 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:14.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2024-08-17T17:49:14.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent 2024-08-17T17:49:14.709 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-08-17T17:49:14.709 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-08-17T17:49:14.717 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2024-08-17T17:49:14.717 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_child 2024-08-17T17:49:14.728 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-08-17T17:49:14.729 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-08-17T17:49:14.735 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-08-17T17:49:14.735 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration 2024-08-17T17:49:14.744 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-08-17T17:49:14.744 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration_v1 2024-08-17T17:49:14.754 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2024-08-17T17:49:14.754 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-08-17T17:49:14.769 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:14.789 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.811 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:14.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:14.912 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:14.933 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:15.035 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:15.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:49:15.056 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:15.157 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:15.178 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:15.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:15.303 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:15.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:15.426 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:15 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:49:15.153+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e502 definitely_dead 0 2024-08-17T17:49:15.528 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:15.550 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:15.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.675 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:15.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:15.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:15.798 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:15.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:15.923 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:16.024 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:16.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.047 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:16.135 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1381 ms) 2024-08-17T17:49:16.135 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparsify 2024-08-17T17:49:16.143 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.1. 2024-08-17T17:49:16.147 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:16.169 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:16.188 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparsify (53 ms) 2024-08-17T17:49:16.271 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:16.293 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:16.394 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:16.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:16 smithi175 ceph-mon[101434]: pgmap v1549: 115 pgs: 9 creating+peering, 10 active+clean+remapped, 22 active+undersized, 13 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 1.8 MiB/s rd, 959 KiB/s wr, 728 op/s; 16/138 objects degraded (11.594%); 5/138 objects misplaced (3.623%) 2024-08-17T17:49:16.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:16 smithi175 ceph-mon[101434]: osdmap e502: 8 total, 7 up, 7 in 2024-08-17T17:49:16.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:16 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1195552387' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:16 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:16 smithi175 ceph-mon[101434]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:49:16.415 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[104516]: pgmap v1549: 115 pgs: 9 creating+peering, 10 active+clean+remapped, 22 active+undersized, 13 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 1.8 MiB/s rd, 959 KiB/s wr, 728 op/s; 16/138 objects degraded (11.594%); 5/138 objects misplaced (3.623%) 2024-08-17T17:49:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[104516]: osdmap e502: 8 total, 7 up, 7 in 2024-08-17T17:49:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1195552387' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[104516]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:49:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[109246]: pgmap v1549: 115 pgs: 9 creating+peering, 10 active+clean+remapped, 22 active+undersized, 13 active+undersized+degraded, 61 active+clean; 4.6 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 1.8 MiB/s rd, 959 KiB/s wr, 728 op/s; 16/138 objects degraded (11.594%); 5/138 objects misplaced (3.623%) 2024-08-17T17:49:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[109246]: osdmap e502: 8 total, 7 up, 7 in 2024-08-17T17:49:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1195552387' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:49:16.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:16 smithi064 ceph-mon[109246]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:49:16.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:16.543 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:16.644 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:16.665 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:16.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:16.789 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:16.891 DEBUG:teuthology.orchestra.run.smithi175:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:16.912 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: docker: command not found 2024-08-17T17:49:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi175 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:17.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:17.036 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:17.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:17.147 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd (5074 ms total) 2024-08-17T17:49:17.147 INFO:tasks.workunit.client.0.smithi064.stdout: 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment tear-down 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] 70 tests from 1 test suite ran. (7757 ms total) 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout:[ PASSED ] 69 tests. 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] 1 test, listed below: 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout: 2024-08-17T17:49:17.148 INFO:tasks.workunit.client.0.smithi064.stdout: 1 FAILED TEST 2024-08-17T17:49:17.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.168 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-08-17T17:49:17.168 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:49:17.186 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:17.203 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:17.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:17.327 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: docker: command not found 2024-08-17T17:49:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 1: 'sudo docker kill -s 1 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[104516]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[104516]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[109246]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:49:17.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:17 smithi064 ceph-mon[109246]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:49:17.416 INFO:tasks.thrashosds:joining thrashosds 2024-08-17T17:49:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:17 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:49:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:17 smithi175 ceph-mon[101434]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:49:17.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:17 smithi175 ceph-mon[101434]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[104516]: pgmap v1552: 115 pgs: 13 peering, 2 unknown, 10 active+clean+remapped, 21 active+undersized, 17 active+undersized+degraded, 52 active+clean; 4.7 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 865 op/s; 21/225 objects degraded (9.333%); 5/225 objects misplaced (2.222%); 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[104516]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[109246]: pgmap v1552: 115 pgs: 13 peering, 2 unknown, 10 active+clean+remapped, 21 active+undersized, 17 active+undersized+degraded, 52 active+clean; 4.7 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 865 op/s; 21/225 objects degraded (9.333%); 5/225 objects misplaced (2.222%); 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-17T17:49:18.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:18 smithi064 ceph-mon[109246]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:49:18.432 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2024-08-17T17:49:18.432 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2024-08-17T17:49:18.432 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2024-08-17T17:49:18.433 INFO:teuthology.orchestra.run.smithi064.stdout:} 2024-08-17T17:49:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:18 smithi175 ceph-mon[101434]: pgmap v1552: 115 pgs: 13 peering, 2 unknown, 10 active+clean+remapped, 21 active+undersized, 17 active+undersized+degraded, 52 active+clean; 4.7 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 865 op/s; 21/225 objects degraded (9.333%); 5/225 objects misplaced (2.222%); 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:18 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-17T17:49:18.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:18 smithi175 ceph-mon[101434]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:49:19.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:49:19.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:19 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 21/225 objects degraded (9.333%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:19.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:19 smithi064 ceph-mon[104516]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:49:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:19 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 21/225 objects degraded (9.333%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:19.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:19 smithi064 ceph-mon[109246]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:49:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:19 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 21/225 objects degraded (9.333%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:19.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:19 smithi175 ceph-mon[101434]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:49:20.523 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[109246]: pgmap v1556: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 15 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 33 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 5/84 objects misplaced (5.952%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[109246]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[109246]: 1.3 scrub ok 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[104516]: pgmap v1556: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 15 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 33 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 5/84 objects misplaced (5.952%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:20.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[104516]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:49:20.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:49:20.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[104516]: 1.3 scrub ok 2024-08-17T17:49:20.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:20 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:20 smithi175 ceph-mon[101434]: pgmap v1556: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 15 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 33 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 5/84 objects misplaced (5.952%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:20 smithi175 ceph-mon[101434]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:49:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:20 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:49:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:20 smithi175 ceph-mon[101434]: 1.3 scrub ok 2024-08-17T17:49:20.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:20 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:21.071 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2024-08-17T17:49:21.072 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:21.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2024-08-17T17:49:21.075 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:21.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2024-08-17T17:49:21.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:21.108 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2024-08-17T17:49:21.108 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:21.126 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2024-08-17T17:49:21.126 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:21.144 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2024-08-17T17:49:21.144 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:21.161 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2024-08-17T17:49:21.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:21.161 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:21.792 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[104516]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[104516]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/602297360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[109246]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[109246]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:49:21.793 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/602297360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:21 smithi175 ceph-mon[101434]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1319483068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:21 smithi175 ceph-mon[101434]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:49:21.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/602297360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:22.008 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:22.195 INFO:tasks.thrashosds.thrasher:reviving osd 2024-08-17T17:49:22.195 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2024-08-17T17:49:22.195 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2024-08-17T17:49:22.195 DEBUG:teuthology.orchestra.run.smithi175:osd.7> sudo journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:49:22.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl start ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:49:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:22.430 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:22.526 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 systemd[1]: Starting Ceph osd.7 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:49:22.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:22 smithi064 ceph-mon[104516]: pgmap v1559: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 19 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 30 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 6/84 objects misplaced (7.143%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:22.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:22 smithi064 ceph-mon[109246]: pgmap v1559: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 19 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 30 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 6/84 objects misplaced (7.143%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:22.847 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:22 smithi175 ceph-mon[101434]: pgmap v1559: 83 pgs: 10 peering, 5 remapped+peering, 2 unknown, 19 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 30 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 6/84 objects misplaced (7.143%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:22.848 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 podman[248214]: 2024-08-17 17:49:22.429057352 +0000 UTC m=+0.017773033 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:49:22.848 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 podman[248214]: 2024-08-17 17:49:22.527948277 +0000 UTC m=+0.116663951 container create 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:49:22.848 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 podman[248214]: 2024-08-17 17:49:22.764033471 +0000 UTC m=+0.352749143 container init 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS) 2024-08-17T17:49:22.848 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 podman[248214]: 2024-08-17 17:49:22.767717733 +0000 UTC m=+0.356433414 container start 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:49:22.848 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:22 smithi175 podman[248214]: 2024-08-17 17:49:22.847209721 +0000 UTC m=+0.435925407 container attach 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:49:23.111 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate[248229]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:49:23.112 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 bash[248214]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:49:23.113 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 podman[248214]: 2024-08-17 17:49:23.111046227 +0000 UTC m=+0.699761898 container died 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:49:23.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:49:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:23 smithi064 ceph-mon[104516]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:23 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:23.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:23 smithi064 ceph-mon[109246]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:23.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:23 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:23.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-mon[101434]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:23.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:23 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 9/84 objects degraded (10.714%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:23.911 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 podman[248357]: 2024-08-17 17:49:23.709041934 +0000 UTC m=+0.591979511 container remove 3ba720fba71b7947580178cbc9562a34445479f9647b071235ed5b7230733b58 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-activate, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:49:24.110 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:23] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-08-17T17:49:24.221 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 podman[248416]: 2024-08-17 17:49:23.834172916 +0000 UTC m=+0.017875165 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:49:24.221 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:23 smithi175 podman[248416]: 2024-08-17 17:49:23.988801605 +0000 UTC m=+0.172503838 container create 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:49:24.335 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:49:24.535 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 podman[248416]: 2024-08-17 17:49:24.22089694 +0000 UTC m=+0.404599187 container init 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:49:24.535 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 podman[248416]: 2024-08-17 17:49:24.224211225 +0000 UTC m=+0.407913458 container start 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-08-17T17:49:24.536 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 ceph-osd[248435]: -- 172.21.15.175:0/3029574527 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55814e6711e0 con 0x55814f3f0000 2024-08-17T17:49:24.536 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 bash[248416]: 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb 2024-08-17T17:49:24.536 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 systemd[1]: Started Ceph osd.7 for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:49:24.536 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:24 smithi175 ceph-mon[101434]: pgmap v1560: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 7 objects/s recovering 2024-08-17T17:49:24.536 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:24 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:24.543 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:24.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:24 smithi064 ceph-mon[104516]: pgmap v1560: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 7 objects/s recovering 2024-08-17T17:49:24.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:24 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:24.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:24 smithi064 ceph-mon[109246]: pgmap v1560: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 7 objects/s recovering 2024-08-17T17:49:24.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:24 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:24.824 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:49:24.763+0000 7eff8e2e1640 -1 Falling back to public interface 2024-08-17T17:49:25.512 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:25.512 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:26.240 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:26 smithi175 ceph-mon[101434]: pgmap v1561: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 5 objects/s recovering 2024-08-17T17:49:26.240 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:26.240 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:26 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:26.332 INFO:teuthology.orchestra.run.smithi175.stderr:no valid command found; 10 closest matches: 2024-08-17T17:49:26.332 INFO:teuthology.orchestra.run.smithi175.stderr:0 2024-08-17T17:49:26.332 INFO:teuthology.orchestra.run.smithi175.stderr:1 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:2 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:abort 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:assert 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:config diff 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:config diff get 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:config get 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:config help [] 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:config set ... 2024-08-17T17:49:26.333 INFO:teuthology.orchestra.run.smithi175.stderr:admin_socket: invalid command 2024-08-17T17:49:26.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[104516]: pgmap v1561: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 5 objects/s recovering 2024-08-17T17:49:26.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:26.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:26.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[109246]: pgmap v1561: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 5 objects/s recovering 2024-08-17T17:49:26.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:26.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:26 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:26.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:49:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:49:26.910 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2024-08-17T17:49:27.210 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:28.128 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2024-08-17T17:49:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:28 smithi175 ceph-mon[101434]: pgmap v1562: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3636570571' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:28.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:28 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[104516]: pgmap v1562: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3636570571' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:28.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[109246]: pgmap v1562: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3636570571' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:28.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:28 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:29.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:49:29.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:29 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:29.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:29 smithi064 ceph-mon[104516]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:49:29.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:29 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:29.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:29 smithi064 ceph-mon[109246]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:49:29.391 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:29.392 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:29.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:29 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:29.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:29 smithi175 ceph-mon[101434]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:49:30.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:30 smithi064 ceph-mon[104516]: pgmap v1564: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:30.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:30 smithi064 ceph-mon[109246]: pgmap v1564: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:30.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:30 smithi175 ceph-mon[101434]: pgmap v1564: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 4 objects/s recovering 2024-08-17T17:49:30.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:49:30.440 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:31.593 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:31.911 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:49:32.134 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:32.258 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:32 smithi175 ceph-mon[101434]: pgmap v1565: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:32.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:32 smithi064 ceph-mon[104516]: pgmap v1565: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:32.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:32 smithi064 ceph-mon[109246]: pgmap v1565: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%); 0 B/s, 2 objects/s recovering 2024-08-17T17:49:33.145 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2024-08-17T17:49:33.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:33 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:33.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:33 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:33.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:33 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:33.942 INFO:teuthology.orchestra.run.smithi175.stderr:no valid command found; 10 closest matches: 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:0 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:1 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:2 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:abort 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:assert 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:bluefs files list 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:bluefs stats 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:bluestore allocator dump block 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:bluestore allocator fragmentation block 2024-08-17T17:49:33.943 INFO:teuthology.orchestra.run.smithi175.stderr:admin_socket: invalid command 2024-08-17T17:49:34.086 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:33] "GET /metrics HTTP/1.1" 200 34359 "" "Prometheus/2.51.0" 2024-08-17T17:49:34.299 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:34.299 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:34 smithi175 ceph-mon[101434]: pgmap v1566: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:34.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:34 smithi175 ceph-mon[101434]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:49:34.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[109246]: pgmap v1566: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:34.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:34.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[109246]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:49:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[104516]: pgmap v1566: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3484700362' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:34.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:34 smithi064 ceph-mon[104516]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:49:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:49:34.687 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2024-08-17T17:49:34.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:49:35.096 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:35.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:49:35.226+0000 7eff8e2e1640 -1 osd.7 485 log_to_monitors true 2024-08-17T17:49:36.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[104516]: pgmap v1568: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[104516]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[104516]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[109246]: pgmap v1568: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[109246]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:36 smithi064 ceph-mon[109246]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:36 smithi175 ceph-mon[101434]: pgmap v1568: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:36 smithi175 ceph-mon[101434]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:36 smithi175 ceph-mon[101434]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:49:36.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:49:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:49:36.198+0000 7eff8608b640 -1 osd.7 511 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:49:36.885 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:36.886 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[104516]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:49:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[104516]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[104516]: osdmap e511: 8 total, 7 up, 6 in 2024-08-17T17:49:37.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[104516]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[109246]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:49:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[109246]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[109246]: osdmap e511: 8 total, 7 up, 6 in 2024-08-17T17:49:37.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:37 smithi064 ceph-mon[109246]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:49:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:37 smithi175 ceph-mon[101434]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:49:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:37 smithi175 ceph-mon[101434]: from='osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:37 smithi175 ceph-mon[101434]: osdmap e511: 8 total, 7 up, 6 in 2024-08-17T17:49:37.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:37 smithi175 ceph-mon[101434]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-08-17T17:49:37.706 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[104516]: pgmap v1570: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[104516]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:49:38.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[104516]: osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703] boot 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[104516]: osdmap e512: 8 total, 8 up, 6 in 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[109246]: pgmap v1570: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[109246]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[109246]: osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703] boot 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[109246]: osdmap e512: 8 total, 8 up, 6 in 2024-08-17T17:49:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:38 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:49:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:38 smithi175 ceph-mon[101434]: pgmap v1570: 83 pgs: 26 active+clean+remapped, 10 active+undersized, 7 active+undersized+degraded, 40 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 9/84 objects degraded (10.714%); 9/84 objects misplaced (10.714%) 2024-08-17T17:49:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:38 smithi175 ceph-mon[101434]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:49:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:38 smithi175 ceph-mon[101434]: osd.7 [v2:172.21.15.175:6824/2836577703,v1:172.21.15.175:6825/2836577703] boot 2024-08-17T17:49:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:38 smithi175 ceph-mon[101434]: osdmap e512: 8 total, 8 up, 6 in 2024-08-17T17:49:38.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:38 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:49:39.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:39 smithi064 ceph-mon[104516]: osdmap e513: 8 total, 8 up, 6 in 2024-08-17T17:49:39.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:39 smithi064 ceph-mon[104516]: osdmap e514: 8 total, 8 up, 6 in 2024-08-17T17:49:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:39 smithi064 ceph-mon[109246]: osdmap e513: 8 total, 8 up, 6 in 2024-08-17T17:49:39.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:39 smithi064 ceph-mon[109246]: osdmap e514: 8 total, 8 up, 6 in 2024-08-17T17:49:39.444 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:39.444 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:39 smithi175 ceph-mon[101434]: osdmap e513: 8 total, 8 up, 6 in 2024-08-17T17:49:39.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:39 smithi175 ceph-mon[101434]: osdmap e514: 8 total, 8 up, 6 in 2024-08-17T17:49:39.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:49:39.920 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:40.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:49:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:40 smithi175 ceph-mon[101434]: pgmap v1573: 83 pgs: 13 peering, 5 remapped+peering, 13 active+clean+remapped, 4 active+undersized, 5 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 7/84 objects degraded (8.333%); 7/84 objects misplaced (8.333%) 2024-08-17T17:49:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:40 smithi175 ceph-mon[101434]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:40.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:40.416 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[104516]: pgmap v1573: 83 pgs: 13 peering, 5 remapped+peering, 13 active+clean+remapped, 4 active+undersized, 5 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 7/84 objects degraded (8.333%); 7/84 objects misplaced (8.333%) 2024-08-17T17:49:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[104516]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:40.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[109246]: pgmap v1573: 83 pgs: 13 peering, 5 remapped+peering, 13 active+clean+remapped, 4 active+undersized, 5 active+undersized+degraded, 43 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 7/84 objects degraded (8.333%); 7/84 objects misplaced (8.333%) 2024-08-17T17:49:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[109246]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:40.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:41.674 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:49:41.674 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:49:41.674 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:49:41.674 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:49:42.127 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:42.127 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:42.320 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T17:49:42.320 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-08-17T17:49:42.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd in 1 2024-08-17T17:49:42.333 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:42 smithi175 ceph-mon[101434]: pgmap v1575: 83 pgs: 13 peering, 5 remapped+peering, 8 active+clean+remapped, 3 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6/84 objects degraded (7.143%); 6/84 objects misplaced (7.143%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:42.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:42 smithi064 ceph-mon[104516]: pgmap v1575: 83 pgs: 13 peering, 5 remapped+peering, 8 active+clean+remapped, 3 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6/84 objects degraded (7.143%); 6/84 objects misplaced (7.143%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:42.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:42 smithi064 ceph-mon[109246]: pgmap v1575: 83 pgs: 13 peering, 5 remapped+peering, 8 active+clean+remapped, 3 active+undersized, 4 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 6/84 objects degraded (7.143%); 6/84 objects misplaced (7.143%); 0 B/s, 0 objects/s recovering 2024-08-17T17:49:42.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:49:42.767 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:43 smithi064 ceph-mon[104516]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded, 7 pgs undersized) 2024-08-17T17:49:43.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:43 smithi064 ceph-mon[109246]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded, 7 pgs undersized) 2024-08-17T17:49:43.518 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:43 smithi175 ceph-mon[101434]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded, 7 pgs undersized) 2024-08-17T17:49:43.956 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:43] "GET /metrics HTTP/1.1" 200 34207 "" "Prometheus/2.51.0" 2024-08-17T17:49:44.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:44 smithi064 ceph-mon[104516]: pgmap v1576: 83 pgs: 8 peering, 75 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:44.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:44 smithi064 ceph-mon[109246]: pgmap v1576: 83 pgs: 8 peering, 75 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:44.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:44 smithi175 ceph-mon[101434]: pgmap v1576: 83 pgs: 8 peering, 75 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:49:45.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:45 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:49:45.321+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e514 definitely_dead 0 2024-08-17T17:49:46.151 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:46.151 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:46.269 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.1. 2024-08-17T17:49:46.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[104516]: pgmap v1577: 83 pgs: 83 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 9.6 KiB/s rd, 28 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:46.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/762616055' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:46.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:46.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[109246]: pgmap v1577: 83 pgs: 83 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 9.6 KiB/s rd, 28 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:46.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/762616055' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:46.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:46 smithi175 ceph-mon[101434]: pgmap v1577: 83 pgs: 83 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 9.6 KiB/s rd, 28 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/762616055' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:46.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:49:47.040 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-08-17T17:49:47.040 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T17:49:47.040 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-08-17T17:49:47.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd in 6 2024-08-17T17:49:47.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:49:47.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[104516]: osdmap e515: 8 total, 8 up, 7 in 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[104516]: 1.9 scrub starts 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[104516]: 1.9 scrub ok 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[109246]: osdmap e515: 8 total, 8 up, 7 in 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[109246]: 1.9 scrub starts 2024-08-17T17:49:47.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:47 smithi064 ceph-mon[109246]: 1.9 scrub ok 2024-08-17T17:49:47.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:49:47.593 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:49:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:47 smithi175 ceph-mon[101434]: osdmap e515: 8 total, 8 up, 7 in 2024-08-17T17:49:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:47 smithi175 ceph-mon[101434]: 1.9 scrub starts 2024-08-17T17:49:47.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:47 smithi175 ceph-mon[101434]: 1.9 scrub ok 2024-08-17T17:49:48.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:48 smithi064 ceph-mon[104516]: pgmap v1579: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:48.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:48 smithi064 ceph-mon[104516]: osdmap e516: 8 total, 8 up, 7 in 2024-08-17T17:49:48.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:48 smithi064 ceph-mon[109246]: pgmap v1579: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:48.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:48 smithi064 ceph-mon[109246]: osdmap e516: 8 total, 8 up, 7 in 2024-08-17T17:49:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:48 smithi175 ceph-mon[101434]: pgmap v1579: 83 pgs: 83 active+clean; 4.6 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:49:48.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:48 smithi175 ceph-mon[101434]: osdmap e516: 8 total, 8 up, 7 in 2024-08-17T17:49:49.077 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:49.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:49 smithi064 ceph-mon[104516]: osdmap e517: 8 total, 8 up, 7 in 2024-08-17T17:49:49.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:49 smithi064 ceph-mon[104516]: osdmap e518: 8 total, 8 up, 7 in 2024-08-17T17:49:49.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:49 smithi064 ceph-mon[109246]: osdmap e517: 8 total, 8 up, 7 in 2024-08-17T17:49:49.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:49 smithi064 ceph-mon[109246]: osdmap e518: 8 total, 8 up, 7 in 2024-08-17T17:49:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:49 smithi175 ceph-mon[101434]: osdmap e517: 8 total, 8 up, 7 in 2024-08-17T17:49:49.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:49 smithi175 ceph-mon[101434]: osdmap e518: 8 total, 8 up, 7 in 2024-08-17T17:49:50.103 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:49:49.703+0000 7fb07c1f8640 -1 mon.a@0(leader).osd e519 definitely_dead 0 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[104516]: pgmap v1583: 83 pgs: 3 remapped+peering, 80 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[104516]: osdmap e519: 8 total, 8 up, 7 in 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[109246]: pgmap v1583: 83 pgs: 3 remapped+peering, 80 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[109246]: osdmap e519: 8 total, 8 up, 7 in 2024-08-17T17:49:50.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T17:49:50.528 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.6. 2024-08-17T17:49:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:50 smithi175 ceph-mon[101434]: pgmap v1583: 83 pgs: 3 remapped+peering, 80 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-08-17T17:49:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:50 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:50 smithi175 ceph-mon[101434]: osdmap e519: 8 total, 8 up, 7 in 2024-08-17T17:49:50.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T17:49:51.295 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-08-17T17:49:51.295 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:49:51.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-08-17T17:49:51.761 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:51.761 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:51 smithi175 ceph-mon[101434]: 1.3 scrub starts 2024-08-17T17:49:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:51 smithi175 ceph-mon[101434]: 1.3 scrub ok 2024-08-17T17:49:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:51 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T17:49:51.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:51 smithi175 ceph-mon[101434]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:49:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[104516]: 1.3 scrub starts 2024-08-17T17:49:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[104516]: 1.3 scrub ok 2024-08-17T17:49:51.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T17:49:51.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[104516]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:49:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[109246]: 1.3 scrub starts 2024-08-17T17:49:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[109246]: 1.3 scrub ok 2024-08-17T17:49:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3588618117' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T17:49:51.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:51 smithi064 ceph-mon[109246]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:49:52.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:49:52.525 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:52.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:52 smithi064 ceph-mon[104516]: pgmap v1586: 83 pgs: 7 peering, 3 remapped+peering, 73 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 57 KiB/s wr, 14 op/s; 41 KiB/s, 1 objects/s recovering 2024-08-17T17:49:52.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:52 smithi064 ceph-mon[104516]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:49:52.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:52 smithi064 ceph-mon[109246]: pgmap v1586: 83 pgs: 7 peering, 3 remapped+peering, 73 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 57 KiB/s wr, 14 op/s; 41 KiB/s, 1 objects/s recovering 2024-08-17T17:49:52.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:52 smithi064 ceph-mon[109246]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:49:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:52 smithi175 ceph-mon[101434]: pgmap v1586: 83 pgs: 7 peering, 3 remapped+peering, 73 active+clean; 4.7 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 19 KiB/s rd, 57 KiB/s wr, 14 op/s; 41 KiB/s, 1 objects/s recovering 2024-08-17T17:49:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:52 smithi175 ceph-mon[101434]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:49:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:53 smithi064 ceph-mon[104516]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:49:53.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:53 smithi064 ceph-mon[104516]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:49:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:53 smithi064 ceph-mon[109246]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:49:53.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:53 smithi064 ceph-mon[109246]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:49:53.826 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:53 smithi175 ceph-mon[101434]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:49:53.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:53 smithi175 ceph-mon[101434]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:49:54.052 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:49:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:49:53] "GET /metrics HTTP/1.1" 200 34207 "" "Prometheus/2.51.0" 2024-08-17T17:49:54.525 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pgp_num to 11 2024-08-17T17:49:54.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[104516]: pgmap v1589: 83 pgs: 1 active+undersized+remapped, 20 peering, 3 remapped+peering, 59 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 5 objects/s recovering 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1926823740' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[104516]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[109246]: pgmap v1589: 83 pgs: 1 active+undersized+remapped, 20 peering, 3 remapped+peering, 59 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 5 objects/s recovering 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1926823740' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:49:54.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:54 smithi064 ceph-mon[109246]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:49:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:54 smithi175 ceph-mon[101434]: pgmap v1589: 83 pgs: 1 active+undersized+remapped, 20 peering, 3 remapped+peering, 59 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 5 objects/s recovering 2024-08-17T17:49:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:54 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1926823740' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:49:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:54 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:49:54.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:54 smithi175 ceph-mon[101434]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:49:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:56 smithi175 ceph-mon[101434]: pgmap v1592: 83 pgs: 4 active+undersized+remapped, 20 peering, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:56 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[104516]: pgmap v1592: 83 pgs: 4 active+undersized+remapped, 20 peering, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[109246]: pgmap v1592: 83 pgs: 4 active+undersized+remapped, 20 peering, 59 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:49:56.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:56 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-08-17T17:49:56.830 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:56.830 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:57.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:57 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:57.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:57 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:57.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:57 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:49:57.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:49:57.618 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:58.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:49:58 smithi175 ceph-mon[101434]: pgmap v1593: 83 pgs: 4 active+undersized+remapped, 79 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:58.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:49:58 smithi064 ceph-mon[104516]: pgmap v1593: 83 pgs: 4 active+undersized+remapped, 79 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:58.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:49:58 smithi064 ceph-mon[109246]: pgmap v1593: 83 pgs: 4 active+undersized+remapped, 79 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2024-08-17T17:49:59.331 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:59.337 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2024-08-17T17:49:59.338 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:49:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2024-08-17T17:49:59.348 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:49:59.357 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2024-08-17T17:49:59.358 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:50:00.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[104516]: pgmap v1594: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/633377145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[104516]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[104516]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[104516]: daemon osd.7 on smithi175 is in unknown state 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[109246]: pgmap v1594: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/633377145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[109246]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[109246]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:00 smithi064 ceph-mon[109246]: daemon osd.7 on smithi175 is in unknown state 2024-08-17T17:50:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:00 smithi175 ceph-mon[101434]: pgmap v1594: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2024-08-17T17:50:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:00 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/633377145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:50:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:00 smithi175 ceph-mon[101434]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-17T17:50:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:00 smithi175 ceph-mon[101434]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:00 smithi175 ceph-mon[101434]: daemon osd.7 on smithi175 is in unknown state 2024-08-17T17:50:01.808 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:01.808 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:01.811 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:02.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:02 smithi175 ceph-mon[101434]: pgmap v1595: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2024-08-17T17:50:02.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:02 smithi064 ceph-mon[104516]: pgmap v1595: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2024-08-17T17:50:02.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:02 smithi064 ceph-mon[109246]: pgmap v1595: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2024-08-17T17:50:03.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:50:04.061 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:04.140 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:03] "GET /metrics HTTP/1.1" 200 34356 "" "Prometheus/2.51.0" 2024-08-17T17:50:04.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:04 smithi175 ceph-mon[101434]: pgmap v1596: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 504 B/s, 2 objects/s recovering 2024-08-17T17:50:04.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:04 smithi064 ceph-mon[104516]: pgmap v1596: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 504 B/s, 2 objects/s recovering 2024-08-17T17:50:04.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:04 smithi064 ceph-mon[109246]: pgmap v1596: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 504 B/s, 2 objects/s recovering 2024-08-17T17:50:05.150 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:50:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:05 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/832401832' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:05 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:05 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/832401832' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:05 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:05 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/832401832' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:05 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:05.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 1 2024-08-17T17:50:06.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[104516]: pgmap v1597: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 457 B/s, 2 objects/s recovering 2024-08-17T17:50:06.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:06.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[104516]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:50:06.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[109246]: pgmap v1597: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 457 B/s, 2 objects/s recovering 2024-08-17T17:50:06.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:06.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:06 smithi064 ceph-mon[109246]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:50:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:06 smithi175 ceph-mon[101434]: pgmap v1597: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 457 B/s, 2 objects/s recovering 2024-08-17T17:50:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:06 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:06.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:06 smithi175 ceph-mon[101434]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:50:06.465 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:06.465 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:06.474 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:07.105 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:50:07.333 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:08.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:08 smithi064 ceph-mon[104516]: pgmap v1599: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:08.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:08 smithi064 ceph-mon[109246]: pgmap v1599: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:08.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:08 smithi175 ceph-mon[101434]: pgmap v1599: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:08.962 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:08.962 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:09.162 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-08-17T17:50:09.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:09 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3313140404' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:09 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:09 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3313140404' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:09 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:09 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3313140404' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:09 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:09.629 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:50:09.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:50:10.277 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: pgmap v1600: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:10.278 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:10.278 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:50:10.278 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:10.278 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:10.278 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:10 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:10.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: pgmap v1600: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: pgmap v1600: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:10.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:50:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:10.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:10 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:10.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:11.172 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:12.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:12 smithi064 ceph-mon[104516]: pgmap v1602: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:12.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:12 smithi064 ceph-mon[109246]: pgmap v1602: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:12.289 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:12 smithi175 ceph-mon[101434]: pgmap v1602: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:12.851 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:12.851 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:13.169 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:50:13.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:13 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:13.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:13 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:13.444 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:13 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:13.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 1 2024-08-17T17:50:14.079 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:13] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-08-17T17:50:14.166 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:50:14.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[104516]: pgmap v1603: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:14.339 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:14.339 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[104516]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:50:14.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[109246]: pgmap v1603: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:14.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:14.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:14 smithi064 ceph-mon[109246]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:50:14.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:14 smithi175 ceph-mon[101434]: pgmap v1603: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:14 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2463473421' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:14.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:14 smithi175 ceph-mon[101434]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:50:14.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:14.893 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:16.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[109246]: pgmap v1605: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:16.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:16.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:16.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[104516]: pgmap v1605: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:16.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:16.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:16 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:16.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:16 smithi175 ceph-mon[101434]: pgmap v1605: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:16 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:16.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:16 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:17.092 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:17.092 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:17.799 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:50:18.005 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.127 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:17 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:17 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:18.215 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-08-17T17:50:18.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:50:19.119 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: pgmap v1606: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: pgmap v1607: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:19.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:18 smithi175 ceph-mon[101434]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:50:19.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: pgmap v1606: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: pgmap v1607: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:19.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[109246]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: pgmap v1606: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: pgmap v1607: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3068589266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:19.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:18 smithi064 ceph-mon[104516]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:50:19.686 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:19.687 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:20.323 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:50:20.527 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:20.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:20 smithi175 ceph-mon[101434]: pgmap v1609: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:20.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:20 smithi064 ceph-mon[104516]: pgmap v1609: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:20.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:20 smithi064 ceph-mon[109246]: pgmap v1609: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:21.853 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:50:21.967 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:21 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3801484308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:21.967 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:21 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:22.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:21 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3801484308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:22.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:21 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:22.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:21 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3801484308' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:22.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:21 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:22.234 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:22.234 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:22.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-17T17:50:22.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:22.931 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:50:23.141 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:23.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[104516]: pgmap v1610: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:23.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:23.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[104516]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:50:23.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[109246]: pgmap v1610: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:23.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:23.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:22 smithi064 ceph-mon[109246]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:50:23.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:22 smithi175 ceph-mon[101434]: pgmap v1610: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:22 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:23.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:22 smithi175 ceph-mon[101434]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:50:24.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:23] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-08-17T17:50:24.780 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:24.780 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:24.856 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-17T17:50:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:24 smithi175 ceph-mon[101434]: pgmap v1612: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:24 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/295203875' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:24 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[104516]: pgmap v1612: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:25.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/295203875' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[109246]: pgmap v1612: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:25.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/295203875' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:24 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:25.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:50:25.541 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:50:25.745 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:25.746 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:25 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:25 smithi175 ceph-mon[101434]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:50:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:26.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:26.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[104516]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[109246]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:50:26.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:27.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:26 smithi064 ceph-mon[104516]: pgmap v1614: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:27.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:26 smithi064 ceph-mon[104516]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:50:27.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:26 smithi064 ceph-mon[109246]: pgmap v1614: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:27.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:26 smithi064 ceph-mon[109246]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:50:27.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:26 smithi175 ceph-mon[101434]: pgmap v1614: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:27.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:26 smithi175 ceph-mon[101434]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:50:27.479 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:27.479 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:27.865 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:50:28.109 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:27 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:28.138 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:50:28.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:27 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:28.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:27 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:28.343 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:28.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-17T17:50:28.827 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:28 smithi175 ceph-mon[101434]: pgmap v1616: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:28 smithi175 ceph-mon[101434]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:28 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:29.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:28 smithi175 ceph-mon[101434]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:50:29.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[104516]: pgmap v1616: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:29.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[104516]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:29.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:29.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[104516]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:50:29.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[109246]: pgmap v1616: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:29.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[109246]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:29.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/460338689' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:29.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:28 smithi064 ceph-mon[109246]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:50:30.061 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:30.062 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:30.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:50:30.871 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-17T17:50:31.067 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:31.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:30 smithi175 ceph-mon[101434]: pgmap v1618: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2761784284' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:30 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[104516]: pgmap v1618: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:31.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2761784284' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[109246]: pgmap v1618: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2761784284' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:30 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:50:31.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:50:32.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:32.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:31 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:32.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:31 smithi175 ceph-mon[101434]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:50:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:31 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:32.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:31 smithi064 ceph-mon[104516]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:50:32.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:31 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:50:32.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:31 smithi064 ceph-mon[109246]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:50:32.716 INFO:teuthology.orchestra.run.smithi175.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:32.716 INFO:teuthology.orchestra.run.smithi175.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:32 smithi175 ceph-mon[101434]: pgmap v1620: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:32 smithi175 ceph-mon[101434]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:50:33.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:32 smithi175 ceph-mon[101434]: Cluster is now healthy 2024-08-17T17:50:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[104516]: pgmap v1620: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[104516]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:50:33.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[104516]: Cluster is now healthy 2024-08-17T17:50:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[109246]: pgmap v1620: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[109246]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:50:33.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:32 smithi064 ceph-mon[109246]: Cluster is now healthy 2024-08-17T17:50:33.386 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:50:33.592 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:34.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:33] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-08-17T17:50:34.889 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:50:34.993 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:34 smithi175 ceph-mon[101434]: pgmap v1621: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:34.993 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:34 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2475341175' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:34.993 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:34 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:35.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[104516]: pgmap v1621: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2475341175' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:35.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[109246]: pgmap v1621: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2475341175' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:35.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:34 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:35.382 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:50:35.382 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:50:35.382 INFO:teuthology.orchestra.run.smithi175.stdout: "num_ops": 0 2024-08-17T17:50:35.382 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:50:35.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-17T17:50:35.816 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:35.991 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:35 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:35.991 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:35 smithi175 ceph-mon[101434]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:50:35.992 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:50:36.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:35 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:36.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:35 smithi064 ceph-mon[104516]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:50:36.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:35 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:36.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:35 smithi064 ceph-mon[109246]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:50:36.196 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:36.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:36 smithi175 ceph-mon[101434]: pgmap v1623: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:37.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:36 smithi064 ceph-mon[104516]: pgmap v1623: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:37.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:36 smithi064 ceph-mon[109246]: pgmap v1623: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:37.880 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:50:37.880 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [], 2024-08-17T17:50:37.880 INFO:teuthology.orchestra.run.smithi175.stdout: "complaint_time": 30, 2024-08-17T17:50:37.880 INFO:teuthology.orchestra.run.smithi175.stdout: "num_blocked_ops": 0 2024-08-17T17:50:37.880 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:50:37.905 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-17T17:50:38.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1501054234' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:37 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1501054234' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:37 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1501054234' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:37 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:50:38.607 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:50:38.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:50:38.810 INFO:teuthology.orchestra.run.smithi175.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:38.866 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[104516]: pgmap v1624: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[104516]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[109246]: pgmap v1624: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:39.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:38 smithi064 ceph-mon[109246]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:50:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:38 smithi175 ceph-mon[101434]: pgmap v1624: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:38 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:50:39.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:38 smithi175 ceph-mon[101434]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:50:40.402 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-08-17T17:50:40.402 INFO:teuthology.orchestra.run.smithi175.stdout: "size": 20, 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 600, 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "ops": [ 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGPush(3.1c 514/513 [PushOp(3:3ed09add:::607.00000000:head, version: 71'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:3ed09add:::607.00000000:head@71'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:39.177743+0000", 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 61.222837714999997, 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00032161499999999998, 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.403 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177743+0000", 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177743+0000", 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177742+0000", 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.404 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177790+0000", 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7339999999999997e-05 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177805+0000", 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.5597999999999999e-05 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.405 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177822+0000", 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7348999999999999e-05 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177894+0000", 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.1678000000000001e-05 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.406 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.177908+0000", 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.342e-05 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.178064+0000", 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000156658 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.407 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGPush(3.6 514/513 [PushOp(3:654134d2:::mds0_openfiles.0:head, version: 71'1, data_included: [], data_size: 0, omap_header_size: 42, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:654134d2:::mds0_openfiles.0:head@71'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:39.182533+0000", 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 61.218047568000003, 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000241625, 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.408 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182533+0000", 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182533+0000", 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182532+0000", 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.409 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182550+0000", 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.7609999999999999e-05 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182566+0000", 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.6733999999999999e-05 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.410 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182594+0000", 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.7900000000000001e-05 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182641+0000", 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6913999999999998e-05 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.411 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182648+0000", 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:39.182774+0000", 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012670699999999999 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.412 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDScrubReserve(1.9 GRANT e515)", 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.203182+0000", 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.197398282000002, 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000185558, 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203182+0000", 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.413 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203182+0000", 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203180+0000", 2024-08-17T17:50:40.414 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999981 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203194+0000", 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3737000000000001e-05 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203221+0000", 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.6936000000000001e-05 2024-08-17T17:50:40.415 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203237+0000", 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.588e-05 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203318+0000", 2024-08-17T17:50:40.416 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1218999999999996e-05 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203353+0000", 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4592999999999999e-05 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203356+0000", 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.2260000000000001e-06 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.417 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203368+0000", 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.171e-05 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDScrubReserve(1.9 GRANT e515)", 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.203221+0000", 2024-08-17T17:50:40.418 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.197359585000001, 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00023264, 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "started", 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203221+0000", 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.419 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203221+0000", 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203219+0000", 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203227+0000", 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3839999999999999e-06 2024-08-17T17:50:40.420 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203245+0000", 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.8479e-05 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203257+0000", 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1836e-05 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.421 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203401+0000", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000144475 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203426+0000", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4227e-05 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "started", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203428+0000", 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.3760000000000002e-06 2024-08-17T17:50:40.422 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.203453+0000", 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5429999999999999e-05 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGPushReply(3.e 516/515 [PushReplyOp(3:717a0223:::608.00000000:head)])", 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.288646+0000", 2024-08-17T17:50:40.423 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.111934482000002, 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019889699999999999, 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288646+0000", 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288646+0000", 2024-08-17T17:50:40.424 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288646+0000", 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288651+0000", 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1669999999999998e-06 2024-08-17T17:50:40.425 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288659+0000", 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.7700000000000007e-06 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288667+0000", 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.3660000000000003e-06 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.426 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288718+0000", 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0908000000000001e-05 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.288845+0000", 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012701699999999999 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.427 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:1e09599c:::rbd_data.39e44fbca114.0000000000000000:head,73'5)", 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.291715+0000", 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.108864908000001, 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00016210000000000001, 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291715+0000", 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.428 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291715+0000", 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291715+0000", 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.429 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291720+0000", 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.9570000000000001e-06 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291729+0000", 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 9.7890000000000006e-06 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291736+0000", 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9210000000000003e-06 2024-08-17T17:50:40.430 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291762+0000", 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.5871e-05 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.291878+0000", 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000115276 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.431 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:021c8c79:::rbd_data.39fcff4eec51.0000000000000002:head,73'7)", 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.292540+0000", 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.108040365999997, 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000189345, 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.432 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292540+0000", 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292540+0000", 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292540+0000", 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.433 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292542+0000", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.526e-06 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292549+0000", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.131e-06 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292553+0000", 2024-08-17T17:50:40.434 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.7809999999999999e-06 2024-08-17T17:50:40.435 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.435 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.435 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292622+0000", 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.9004999999999994e-05 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.292729+0000", 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010730600000000001 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.436 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:0c180642:::rbd_header.3a7a2735b2ab:head,77'27)", 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.294156+0000", 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.106424642, 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019101900000000001, 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294156+0000", 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.437 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294156+0000", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294155+0000", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294160+0000", 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.1399999999999999e-06 2024-08-17T17:50:40.438 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294173+0000", 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2685e-05 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294178+0000", 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.0980000000000001e-06 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.439 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294241+0000", 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3291000000000001e-05 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294347+0000", 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105544 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:11381167:::rbd_object_map.3ad49d5b8406:head,77'44)", 2024-08-17T17:50:40.440 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.294938+0000", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.105642629000002, 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00017929799999999999, 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294938+0000", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294938+0000", 2024-08-17T17:50:40.441 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294937+0000", 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294941+0000", 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.3859999999999999e-06 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.442 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294949+0000", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.1920000000000005e-06 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.294970+0000", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.126e-05 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295005+0000", 2024-08-17T17:50:40.443 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4817000000000002e-05 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295117+0000", 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00011220100000000001 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:08e1a037:::rbd_data.6230bc520ed.0000000000000000:head,85'48)", 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.295708+0000", 2024-08-17T17:50:40.444 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.104872809, 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015230699999999999, 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295708+0000", 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295708+0000", 2024-08-17T17:50:40.445 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295707+0000", 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295713+0000", 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.446 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295726+0000", 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3135e-05 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295732+0000", 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5740000000000003e-06 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.447 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295755+0000", 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.2900000000000001e-05 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.295860+0000", 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000104955 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.448 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:0b3f613c:::rbd_data.628a60393535.0000000000000000:head,85'67)", 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.296451+0000", 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.104129620000002, 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015514899999999999, 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296451+0000", 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.449 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296451+0000", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296450+0000", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296455+0000", 2024-08-17T17:50:40.450 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3109999999999996e-06 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296469+0000", 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3406e-05 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296474+0000", 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.3739999999999997e-06 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.451 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296499+0000", 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4837e-05 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.296606+0000", 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010699 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.452 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:121fede8:::rbd_header.628a60393535:head,85'70)", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.297197+0000", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.103382893000003, 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019151500000000001, 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297197+0000", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.453 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297197+0000", 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297197+0000", 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297202+0000", 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.057e-06 2024-08-17T17:50:40.454 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297214+0000", 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.2113000000000001e-05 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297219+0000", 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.237e-06 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.455 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297283+0000", 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.3553000000000006e-05 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297389+0000", 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.0001063 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e516,515 2:1ac60794:::rbd_header.62f92c59c3c7:head,85'85)", 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:47.297956+0000", 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 53.102624511999998, 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00015905800000000001, 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.457 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297956+0000", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297956+0000", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297956+0000", 2024-08-17T17:50:40.458 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967296 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297958+0000", 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.922e-06 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297962+0000", 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6999999999999999e-06 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.459 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.297966+0000", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.5549999999999998e-06 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.298009+0000", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3631999999999998e-05 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:47.298115+0000", 2024-08-17T17:50:40.460 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000105407 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGPushReply(3.d 521/520 [PushReplyOp(3:b50e409b:::mds_snaptable:head)])", 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.568649+0000", 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.831931412000003, 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000199392, 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.461 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568649+0000", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568649+0000", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568649+0000", 2024-08-17T17:50:40.462 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967296 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568651+0000", 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 2.4449999999999999e-06 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568658+0000", 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.5089999999999999e-06 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.463 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568663+0000", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.5759999999999996e-06 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568707+0000", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.3736999999999998e-05 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.568848+0000", 2024-08-17T17:50:40.464 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00014127000000000001 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:f17fb142:::rbd_data.39fcff4eec51.00000000000000f4:head,73'3)", 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.570665+0000", 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.829915118000002, 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027694500000000001, 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.465 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570665+0000", 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570665+0000", 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570665+0000", 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.466 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570670+0000", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2599999999999996e-06 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570681+0000", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1647e-05 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570689+0000", 2024-08-17T17:50:40.467 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.9019999999999999e-06 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570736+0000", 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6335000000000003e-05 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.570942+0000", 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00020648899999999999 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.468 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:e55da81e:::rbd_object_map.3a1a436f3a50:head,77'24)", 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.571602+0000", 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.828978352999997, 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00026119199999999999, 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571602+0000", 2024-08-17T17:50:40.469 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571602+0000", 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571601+0000", 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.470 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571607+0000", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4410000000000001e-06 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571618+0000", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.1514e-05 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571624+0000", 2024-08-17T17:50:40.471 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8810000000000001e-06 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571665+0000", 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.1198000000000001e-05 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.571863+0000", 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00019789300000000001 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.472 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:e83de9af:::rbd_data.615bd95d995b.0000000000000000:head,77'28)", 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.572584+0000", 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.827996355000003, 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00027955900000000002, 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572584+0000", 2024-08-17T17:50:40.473 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572584+0000", 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572583+0000", 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.474 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572590+0000", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.0620000000000003e-06 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572603+0000", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3064000000000001e-05 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572612+0000", 2024-08-17T17:50:40.475 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 8.5609999999999999e-06 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572656+0000", 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.4350000000000001e-05 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.572864+0000", 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00020729 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.476 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:e5bceeec:::rbd_data.3ac8571145ab.0000000000000019:head,77'46)", 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.573581+0000", 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.826999196999999, 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000201409, 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573581+0000", 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573581+0000", 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573580+0000", 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.478 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573586+0000", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.4999999999999999e-06 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573599+0000", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.3219e-05 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573607+0000", 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 7.7409999999999992e-06 2024-08-17T17:50:40.479 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573654+0000", 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7231000000000002e-05 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.573783+0000", 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00012853800000000001 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.480 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:ff2d81b8:::rbd_header.3ac8571145ab:head,77'49)", 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.574561+0000", 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.826019148999997, 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00025342700000000003, 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574561+0000", 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.481 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574561+0000", 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574560+0000", 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967295.999999 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.482 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574566+0000", 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.2120000000000002e-06 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574577+0000", 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 1.132e-05 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574584+0000", 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 6.8669999999999996e-06 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.483 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574631+0000", 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.6737999999999999e-05 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.574815+0000", 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00018402300000000001 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.484 INFO:teuthology.orchestra.run.smithi175.stdout: "description": "MOSDPGRecoveryDeleteReply(2.7 e521,520 2:fb688810:::rbd_header.6109fc6c8d91:head,85'68)", 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "initiated_at": "2024-08-17T17:49:51.575457+0000", 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "age": 48.825123232000003, 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.000155513, 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "type_data": { 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "flag_point": "reached pg", 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "events": [ 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "initiated", 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575457+0000", 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.485 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "throttled", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575457+0000", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "header_read", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575457+0000", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4294967296 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "all_read", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575460+0000", 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.4910000000000001e-06 2024-08-17T17:50:40.486 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "dispatched", 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575465+0000", 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 4.7129999999999996e-06 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "queued_for_pg", 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575471+0000", 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 5.8039999999999998e-06 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "reached_pg", 2024-08-17T17:50:40.487 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575508+0000", 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 3.6850000000000001e-05 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: }, 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: { 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: "event": "done", 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: "time": "2024-08-17T17:49:51.575613+0000", 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: "duration": 0.00010480399999999999 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: } 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-08-17T17:50:40.488 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-08-17T17:50:40.902 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:50:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:40 smithi175 ceph-mon[101434]: pgmap v1626: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:40 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:41.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[104516]: pgmap v1626: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:41.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:41.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[109246]: pgmap v1626: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:41.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:40 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:41.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T17:50:41.783 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:41 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:41 smithi064 ceph-mon[104516]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:50:42.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:41 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:42.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:41 smithi064 ceph-mon[109246]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:50:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:41 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3697447343' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:42.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:41 smithi175 ceph-mon[101434]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:50:43.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:42 smithi064 ceph-mon[104516]: pgmap v1628: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:43.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:42 smithi064 ceph-mon[109246]: pgmap v1628: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:43.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:42 smithi175 ceph-mon[101434]: pgmap v1628: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:43.927 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T17:50:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:43 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3642065668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:43 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.188 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:43] "GET /metrics HTTP/1.1" 200 34356 "" "Prometheus/2.51.0" 2024-08-17T17:50:44.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:43 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3642065668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:43 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:43 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3642065668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:43 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:50:44.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:50:44.928 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:44.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[109246]: pgmap v1629: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:44.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:44.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[109246]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:50:44.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[104516]: pgmap v1629: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:44.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:44.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:44 smithi064 ceph-mon[104516]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:50:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:44 smithi175 ceph-mon[101434]: pgmap v1629: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:44 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:50:45.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:44 smithi175 ceph-mon[101434]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:50:46.931 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:50:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[104516]: pgmap v1631: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1738256224' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[109246]: pgmap v1631: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1738256224' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:46 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:46 smithi175 ceph-mon[101434]: pgmap v1631: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:46 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1738256224' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:46 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:47.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T17:50:47.936 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:47.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:47 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:47.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:47 smithi064 ceph-mon[104516]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:50:48.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:47 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:48.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:47 smithi064 ceph-mon[109246]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:50:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:47 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:48.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:47 smithi175 ceph-mon[101434]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:50:49.042 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:48 smithi064 ceph-mon[109246]: pgmap v1633: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:49.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:48 smithi064 ceph-mon[104516]: pgmap v1633: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:49.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:48 smithi175 ceph-mon[101434]: pgmap v1633: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:50:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=cleanup t=2024-08-17T17:50:48.906358593Z level=info msg="Completed cleanup jobs" duration=19.233555ms 2024-08-17T17:50:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:50:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana.update.checker t=2024-08-17T17:50:49.03459028Z level=info msg="Update check succeeded" duration=69.45632ms 2024-08-17T17:50:49.161 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:50:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=plugins.update.checker t=2024-08-17T17:50:49.065317163Z level=info msg="Update check succeeded" duration=73.310839ms 2024-08-17T17:50:49.951 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T17:50:50.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:49 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:50.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:49 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:50.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:49 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:50:50.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:50:50.927 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:50.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[104516]: pgmap v1634: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:50.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:50.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[104516]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:50:50.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[109246]: pgmap v1634: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:50.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:50.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:50 smithi064 ceph-mon[109246]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:50:51.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:50 smithi175 ceph-mon[101434]: pgmap v1634: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:51.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:50 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3551234663' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:50:51.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:50 smithi175 ceph-mon[101434]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:50:52.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:51 smithi064 ceph-mon[104516]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:50:52.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:51 smithi064 ceph-mon[109246]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:50:52.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:51 smithi175 ceph-mon[101434]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:50:52.977 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:50:53.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[104516]: pgmap v1637: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:53.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1375288959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[109246]: pgmap v1637: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:53.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1375288959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:52 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:52 smithi175 ceph-mon[101434]: pgmap v1637: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:53.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:52 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1375288959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:52 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:53.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 7 1 2024-08-17T17:50:53.834 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:54.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:53 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:54.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:53 smithi064 ceph-mon[104516]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:50:54.200 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:50:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:50:53] "GET /metrics HTTP/1.1" 200 34356 "" "Prometheus/2.51.0" 2024-08-17T17:50:54.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:53 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:54.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:53 smithi064 ceph-mon[109246]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:50:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:53 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:54.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:53 smithi175 ceph-mon[101434]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:50:55.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:54 smithi064 ceph-mon[104516]: pgmap v1639: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:55.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:54 smithi064 ceph-mon[109246]: pgmap v1639: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:55.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:54 smithi175 ceph-mon[101434]: pgmap v1639: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-08-17T17:50:56.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:56.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1585239882' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:56.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1585239882' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:55 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:55 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:55 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1585239882' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:55 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:50:56.751 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-08-17T17:50:56.751 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-08-17T17:50:56.751 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-08-17T17:50:56.751 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-08-17T17:50:56.751 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:50:56.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:50:56.973 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:56.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[109246]: pgmap v1640: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:56.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[109246]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:56.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[109246]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:50:57.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[104516]: pgmap v1640: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:57.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[104516]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:57.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:56 smithi064 ceph-mon[104516]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:50:57.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:56 smithi175 ceph-mon[101434]: pgmap v1640: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:57.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:56 smithi175 ceph-mon[101434]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:50:57.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:56 smithi175 ceph-mon[101434]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:50:58.778 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:50:58.778 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":542,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:50:55.987927+0000","last_up_change":"2024-08-17T17:49:37.155347+0000","last_in_change":"2024-08-17T17:49:50.523403+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"524","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"367","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":523,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6803","nonce":2410633150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6805","nonce":2410633150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6809","nonce":2410633150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6807","nonce":2410633150}]},"public_addr":"172.21.15.64:6803/2410633150","cluster_addr":"172.21.15.64:6805/2410633150","heartbeat_back_addr":"172.21.15.64:6809/2410633150","heartbeat_front_addr":"172.21.15.64:6807/2410633150","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":523,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6811","nonce":3737445757}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6813","nonce":3737445757}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6817","nonce":3737445757}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6815","nonce":3737445757}]},"public_addr":"172.21.15.64:6811/3737445757","cluster_addr":"172.21.15.64:6813/3737445757","heartbeat_back_addr":"172.21.15.64:6817/3737445757","heartbeat_front_addr":"172.21.15.64:6815/3737445757","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":530,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6819","nonce":3328029035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6821","nonce":3328029035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6825","nonce":3328029035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6823","nonce":3328029035}]},"public_addr":"172.21.15.64:6819/3328029035","cluster_addr":"172.21.15.64:6821/3328029035","heartbeat_back_addr":"172.21.15.64:6825/3328029035","heartbeat_front_addr":"172.21.15.64:6823/3328029035","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":105,"up_from":109,"up_thru":523,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6827","nonce":2515263413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6829","nonce":2515263413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6833","nonce":2515263413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6831","nonce":2515263413}]},"public_addr":"172.21.15.64:6827/2515263413","cluster_addr":"172.21.15.64:6829/2515263413","heartbeat_back_addr":"172.21.15.64:6833/2515263413","heartbeat_front_addr":"172.21.15.64:6831/2515263413","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":236,"up_from":242,"up_thru":523,"down_at":237,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6809","nonce":1907651607}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6811","nonce":1907651607}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6815","nonce":1907651607}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6813","nonce":1907651607}]},"public_addr":"172.21.15.175:6809/1907651607","cluster_addr":"172.21.15.175:6811/1907651607","heartbeat_back_addr":"172.21.15.175:6815/1907651607","heartbeat_front_addr":"172.21.15.175:6813/1907651607","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":199,"up_from":247,"up_thru":539,"down_at":200,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6817","nonce":1826486060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6819","nonce":1826486060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6823","nonce":1826486060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6821","nonce":1826486060}]},"public_addr":"172.21.15.175:6817/1826486060","cluster_addr":"172.21.15.175:6819/1826486060","heartbeat_back_addr":"172.21.15.175:6823/1826486060","heartbeat_front_addr":"172.21.15.175:6821/1826486060","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":485,"up_from":512,"up_thru":523,"down_at":486,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6825","nonce":2836577703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6827","nonce":2836577703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6831","nonce":2836577703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6829","nonce":2836577703}]},"public_addr":"172.21.15.175:6825/2836577703","cluster_addr":"172.21.15.175:6827/2836577703","heartbeat_back_addr":"172.21.15.175:6831/2836577703","heartbeat_front_addr":"172.21.15.175:6829/2836577703","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":236},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":199},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":485}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.3","mappings":[{"from":4,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/327251329":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6801/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/3929894796":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/4035630069":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1135961297":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/57613042":"2024-08-18T17:09:04.985494+0000","172.21.15.175:0/1707444063":"2024-08-18T17:09:04.985494+0000","172.21.15.175:6834/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/1031286826":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/2705070003":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/4139680086":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6800/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/1358035766":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.175:6835/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2821917932":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.175:0/2662913714":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/3398196717":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.175:0/2315698367":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6801/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:50:59.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:59 smithi064 ceph-mon[104516]: pgmap v1642: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:59.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:50:59 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2798030613' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:50:59.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:59 smithi064 ceph-mon[109246]: pgmap v1642: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:59.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:50:59 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2798030613' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:50:59.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:59 smithi175 ceph-mon[101434]: pgmap v1642: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:50:59.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:50:59 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2798030613' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:50:59.604 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:50:59.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:50:59.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:01.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:01 smithi064 ceph-mon[104516]: pgmap v1643: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:01.263 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:01 smithi064 ceph-mon[109246]: pgmap v1643: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:01.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:01 smithi175 ceph-mon[101434]: pgmap v1643: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:51:01.804 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":542,"fsid":"d49b6e42-5cb9-11ef-bccf-c7b262605968","created":"2024-08-17T16:58:44.445222+0000","modified":"2024-08-17T17:50:55.987927+0000","last_up_change":"2024-08-17T17:49:37.155347+0000","last_in_change":"2024-08-17T17:49:50.523403+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:01:47.759503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"524","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"367","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:04:23.242221+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:05:32.826101+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:05:33.489576+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"88904c1f-eac1-41db-b79a-1a4967d1dae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":523,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6803","nonce":2410633150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6805","nonce":2410633150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6809","nonce":2410633150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2410633150},{"type":"v1","addr":"172.21.15.64:6807","nonce":2410633150}]},"public_addr":"172.21.15.64:6803/2410633150","cluster_addr":"172.21.15.64:6805/2410633150","heartbeat_back_addr":"172.21.15.64:6809/2410633150","heartbeat_front_addr":"172.21.15.64:6807/2410633150","state":["exists","up"]},{"osd":1,"uuid":"9d08169c-5914-4c23-acde-0607da1c3551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":523,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6811","nonce":3737445757}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6813","nonce":3737445757}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6817","nonce":3737445757}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3737445757},{"type":"v1","addr":"172.21.15.64:6815","nonce":3737445757}]},"public_addr":"172.21.15.64:6811/3737445757","cluster_addr":"172.21.15.64:6813/3737445757","heartbeat_back_addr":"172.21.15.64:6817/3737445757","heartbeat_front_addr":"172.21.15.64:6815/3737445757","state":["exists","up"]},{"osd":2,"uuid":"870e8cbe-170a-40f5-8482-de1f6ef4061e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":530,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6819","nonce":3328029035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6821","nonce":3328029035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6825","nonce":3328029035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3328029035},{"type":"v1","addr":"172.21.15.64:6823","nonce":3328029035}]},"public_addr":"172.21.15.64:6819/3328029035","cluster_addr":"172.21.15.64:6821/3328029035","heartbeat_back_addr":"172.21.15.64:6825/3328029035","heartbeat_front_addr":"172.21.15.64:6823/3328029035","state":["exists","up"]},{"osd":3,"uuid":"bad72588-3314-4260-ab5d-2b909812b54b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":105,"up_from":109,"up_thru":523,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6827","nonce":2515263413}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6829","nonce":2515263413}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6833","nonce":2515263413}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":2515263413},{"type":"v1","addr":"172.21.15.64:6831","nonce":2515263413}]},"public_addr":"172.21.15.64:6827/2515263413","cluster_addr":"172.21.15.64:6829/2515263413","heartbeat_back_addr":"172.21.15.64:6833/2515263413","heartbeat_front_addr":"172.21.15.64:6831/2515263413","state":["exists","up"]},{"osd":4,"uuid":"d8a51704-ce10-47b2-806f-ee7a6c85aab0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":523,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6801","nonce":3809583260}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6803","nonce":3809583260}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6807","nonce":3809583260}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":3809583260},{"type":"v1","addr":"172.21.15.175:6805","nonce":3809583260}]},"public_addr":"172.21.15.175:6801/3809583260","cluster_addr":"172.21.15.175:6803/3809583260","heartbeat_back_addr":"172.21.15.175:6807/3809583260","heartbeat_front_addr":"172.21.15.175:6805/3809583260","state":["exists","up"]},{"osd":5,"uuid":"1ebe3c8b-daa3-49aa-a503-4671f0f97c95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":236,"up_from":242,"up_thru":523,"down_at":237,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6809","nonce":1907651607}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6811","nonce":1907651607}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6815","nonce":1907651607}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":1907651607},{"type":"v1","addr":"172.21.15.175:6813","nonce":1907651607}]},"public_addr":"172.21.15.175:6809/1907651607","cluster_addr":"172.21.15.175:6811/1907651607","heartbeat_back_addr":"172.21.15.175:6815/1907651607","heartbeat_front_addr":"172.21.15.175:6813/1907651607","state":["exists","up"]},{"osd":6,"uuid":"6bdf0ddc-f45b-4db6-9a7a-1769117b5b83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":199,"up_from":247,"up_thru":539,"down_at":200,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6817","nonce":1826486060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6819","nonce":1826486060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6823","nonce":1826486060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":1826486060},{"type":"v1","addr":"172.21.15.175:6821","nonce":1826486060}]},"public_addr":"172.21.15.175:6817/1826486060","cluster_addr":"172.21.15.175:6819/1826486060","heartbeat_back_addr":"172.21.15.175:6823/1826486060","heartbeat_front_addr":"172.21.15.175:6821/1826486060","state":["exists","up"]},{"osd":7,"uuid":"1d6aec1a-de13-4aa2-a0cf-5533286fcce6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":485,"up_from":512,"up_thru":523,"down_at":486,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6825","nonce":2836577703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6827","nonce":2836577703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6831","nonce":2836577703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":2836577703},{"type":"v1","addr":"172.21.15.175:6829","nonce":2836577703}]},"public_addr":"172.21.15.175:6825/2836577703","cluster_addr":"172.21.15.175:6827/2836577703","heartbeat_back_addr":"172.21.15.175:6831/2836577703","heartbeat_front_addr":"172.21.15.175:6829/2836577703","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:00:57.344940+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:20.901821+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:01:44.469843+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:10.113124+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:31.369751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:02:53.990564+0000","dead_epoch":236},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:17.539529+0000","dead_epoch":199},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:03:41.546541+0000","dead_epoch":485}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.3","mappings":[{"from":4,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.175:0/4035630069":"2024-08-18T17:09:04.985494+0000","172.21.15.175:0/57613042":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/3398196717":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/3794546409":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/267045601":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6801/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/541572191":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2890874446":"2024-08-18T16:59:37.262677+0000","172.21.15.64:0/1135961297":"2024-08-18T17:08:25.621451+0000","172.21.15.175:0/1707444063":"2024-08-18T17:09:04.985494+0000","172.21.15.175:6834/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/1031286826":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/4139680086":"2024-08-18T17:08:25.621451+0000","172.21.15.64:6800/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/3789040651":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/3939830898":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/3929894796":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/1358035766":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6800/3625104862":"2024-08-18T17:08:25.621451+0000","172.21.15.64:0/327251329":"2024-08-18T17:11:24.558537+0000","172.21.15.175:0/2315698367":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2682323595":"2024-08-18T16:59:37.262677+0000","172.21.15.175:6835/2881274172":"2024-08-18T17:09:04.985494+0000","172.21.15.64:0/2830091420":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/2821917932":"2024-08-18T17:11:24.558537+0000","172.21.15.64:6800/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.175:0/2662913714":"2024-08-18T17:09:04.985494+0000","172.21.15.64:6801/1140276429":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/2705070003":"2024-08-18T17:11:24.558537+0000","172.21.15.64:0/2538331580":"2024-08-18T16:59:13.244932+0000","172.21.15.64:6801/1209068324":"2024-08-18T16:59:13.244932+0000","172.21.15.64:0/1430444926":"2024-08-18T17:04:10.165890+0000","172.21.15.64:0/1313240267":"2024-08-18T17:04:10.165890+0000","172.21.15.64:6801/3078753448":"2024-08-18T16:59:37.262677+0000","172.21.15.64:6800/1140276429":"2024-08-18T17:04:10.165890+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:51:02.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:02 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/84031421' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:51:02.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:02 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/84031421' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:51:02.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:02 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/84031421' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:51:02.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:51:02.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:51:02.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:51:02.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:51:02.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:51:02.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:51:02.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:51:02.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:51:03.118 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:03.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:03 smithi064 ceph-mon[104516]: pgmap v1644: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:03.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:03 smithi064 ceph-mon[109246]: pgmap v1644: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:03.384 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:03.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:03 smithi175 ceph-mon[101434]: pgmap v1644: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:03.854 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:04.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:51:03] "GET /metrics HTTP/1.1" 200 34360 "" "Prometheus/2.51.0" 2024-08-17T17:51:04.686 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:04.687 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:04.690 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:05.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:05 smithi064 ceph-mon[104516]: pgmap v1645: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:05.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:05 smithi064 ceph-mon[109246]: pgmap v1645: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:05.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:05 smithi175 ceph-mon[101434]: pgmap v1645: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:05.861 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:07.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:07 smithi064 ceph-mon[104516]: pgmap v1646: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:07.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:07 smithi064 ceph-mon[109246]: pgmap v1646: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:07.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:07 smithi175 ceph-mon[101434]: pgmap v1646: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:09.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:09 smithi064 ceph-mon[104516]: pgmap v1647: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:09.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:09 smithi064 ceph-mon[109246]: pgmap v1647: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:09.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:09 smithi175 ceph-mon[101434]: pgmap v1647: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:11 smithi064 ceph-mon[104516]: pgmap v1648: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:11.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:11 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:11.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:11 smithi064 ceph-mon[109246]: pgmap v1648: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:11.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:11 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:11 smithi175 ceph-mon[101434]: pgmap v1648: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:11.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:11 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:13.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:13 smithi064 ceph-mon[104516]: pgmap v1649: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:13.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:13 smithi064 ceph-mon[109246]: pgmap v1649: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:13.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:13 smithi175 ceph-mon[101434]: pgmap v1649: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:14.112 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:13 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:51:13] "GET /metrics HTTP/1.1" 200 34356 "" "Prometheus/2.51.0" 2024-08-17T17:51:14.138 INFO:teuthology.orchestra.run.smithi064.stdout:403726926233 2024-08-17T17:51:14.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:51:14.746 INFO:teuthology.orchestra.run.smithi064.stdout:468151435654 2024-08-17T17:51:14.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:51:14.934 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:15.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:15 smithi175 ceph-mon[101434]: pgmap v1650: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:15.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:15 smithi064 ceph-mon[104516]: pgmap v1650: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:15.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:15 smithi064 ceph-mon[109246]: pgmap v1650: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:15.460 INFO:teuthology.orchestra.run.smithi064.stdout:446676599181 2024-08-17T17:51:15.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:51:17.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:17 smithi064 ceph-mon[104516]: pgmap v1651: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:17.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:17 smithi064 ceph-mon[109246]: pgmap v1651: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:17.379 INFO:teuthology.orchestra.run.smithi064.stdout:425201762708 2024-08-17T17:51:17.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:51:17.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:17 smithi175 ceph-mon[101434]: pgmap v1651: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:17.921 INFO:teuthology.orchestra.run.smithi064.stdout:133143986762 2024-08-17T17:51:17.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:51:18.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:18.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:18 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.189 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.411 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:18 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:18.713 INFO:teuthology.orchestra.run.smithi064.stdout:1039382085874 2024-08-17T17:51:18.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:51:19.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:19 smithi064 ceph-mon[104516]: pgmap v1652: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:19.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:19 smithi064 ceph-mon[109246]: pgmap v1652: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:19.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:19 smithi175 ceph-mon[101434]: pgmap v1652: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:20.166 INFO:teuthology.orchestra.run.smithi064.stdout:2199023255574 2024-08-17T17:51:20.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:51:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:21 smithi064 ceph-mon[104516]: pgmap v1653: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:21.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:21 smithi064 ceph-mon[109246]: pgmap v1653: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:21.410 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:21 smithi175 ceph-mon[101434]: pgmap v1653: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:21.649 INFO:teuthology.orchestra.run.smithi064.stdout:403726926235 2024-08-17T17:51:22.169 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[104516]: pgmap v1654: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1026449549' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[109246]: pgmap v1654: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:22 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1026449549' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:51:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:22 smithi175 ceph-mon[101434]: pgmap v1654: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:22 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:22.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:22 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1026449549' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:51:23.102 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926233 got 403726926235 for osd.0 2024-08-17T17:51:23.103 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:24.098 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:24.099 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:24.102 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:24.103 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:24.104 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:24.236 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:51:23] "GET /metrics HTTP/1.1" 200 34356 "" "Prometheus/2.51.0" 2024-08-17T17:51:24.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:24 smithi175 ceph-mon[101434]: pgmap v1655: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:24.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:24 smithi064 ceph-mon[104516]: pgmap v1655: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:24.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:24 smithi064 ceph-mon[109246]: pgmap v1655: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:25.224 INFO:teuthology.orchestra.run.smithi064.stdout:1060856922351 2024-08-17T17:51:25.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:51:25.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:25 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:25.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:25 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:25.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:25 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:26.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:26 smithi064 ceph-mon[104516]: pgmap v1656: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:26.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:26 smithi064 ceph-mon[109246]: pgmap v1656: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:26.660 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:26 smithi175 ceph-mon[101434]: pgmap v1656: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:51:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-17T17:51:27.661 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:27 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-08-17T17:51:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:51:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-17T17:51:27.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-08-17T17:51:27.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:51:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-17T17:51:27.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:27 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-08-17T17:51:28.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[104516]: pgmap v1657: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[104516]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[109246]: pgmap v1657: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-17T17:51:28.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:28 smithi064 ceph-mon[109246]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:51:28.754 INFO:teuthology.orchestra.run.smithi064.stdout:468151435657 2024-08-17T17:51:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:28 smithi175 ceph-mon[101434]: pgmap v1657: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:28 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:51:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:28 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-17T17:51:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:28 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-17T17:51:28.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:28 smithi175 ceph-mon[101434]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:51:29.738 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:29 smithi064 ceph-mon[104516]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:51:29.738 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:29 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2346401687' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:51:29.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:29 smithi064 ceph-mon[109246]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:51:29.739 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:29 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2346401687' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:51:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:29 smithi175 ceph-mon[101434]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:51:29.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:29 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2346401687' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:51:30.254 INFO:teuthology.orchestra.run.smithi064.stdout:133143986766 2024-08-17T17:51:30.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:30 smithi064 ceph-mon[104516]: pgmap v1660: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:30.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:30 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3660560792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:51:30.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:30 smithi064 ceph-mon[109246]: pgmap v1660: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:30.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:30 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3660560792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:51:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:30 smithi175 ceph-mon[101434]: pgmap v1660: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:30.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:30 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3660560792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:51:32.035 INFO:teuthology.orchestra.run.smithi064.stdout:425201762711 2024-08-17T17:51:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:32 smithi064 ceph-mon[104516]: pgmap v1661: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:32 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1107095232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:51:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:32 smithi064 ceph-mon[109246]: pgmap v1661: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:32 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1107095232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:51:32.682 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762708 got 425201762711 for osd.1 2024-08-17T17:51:32.682 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:32.728 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:32 smithi175 ceph-mon[101434]: pgmap v1661: 83 pgs: 1 activating, 1 peering, 81 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:32.728 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:32 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1107095232' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:51:34.334 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:33 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:51:33] "GET /metrics HTTP/1.1" 200 34353 "" "Prometheus/2.51.0" 2024-08-17T17:51:34.620 INFO:teuthology.orchestra.run.smithi064.stdout:446676599185 2024-08-17T17:51:34.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:34 smithi064 ceph-mon[104516]: pgmap v1662: 83 pgs: 1 activating, 82 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:34.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:34 smithi064 ceph-mon[109246]: pgmap v1662: 83 pgs: 1 activating, 82 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:34.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:34 smithi175 ceph-mon[101434]: pgmap v1662: 83 pgs: 1 activating, 82 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:35.280 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676599181 got 446676599185 for osd.2 2024-08-17T17:51:35.280 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:35.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:35 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1584475841' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:51:35.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:35 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1584475841' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:51:35.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:35 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1584475841' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:51:36.140 INFO:teuthology.orchestra.run.smithi064.stdout:1039382085877 2024-08-17T17:51:36.576 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:36.763 INFO:teuthology.orchestra.run.smithi064.stdout:2199023255577 2024-08-17T17:51:36.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:36 smithi064 ceph-mon[104516]: pgmap v1663: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:36.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:36 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/2274660709' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:51:36.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:36 smithi064 ceph-mon[109246]: pgmap v1663: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:36.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:36 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/2274660709' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:51:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:36 smithi175 ceph-mon[101434]: pgmap v1663: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:36.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:36 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/2274660709' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:51:37.158 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986762 got 133143986766 for osd.4 2024-08-17T17:51:37.158 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:37.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:37 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/3237814219' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:51:37.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:37 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/3237814219' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:51:37.881 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1039382085874 got 1039382085877 for osd.5 2024-08-17T17:51:37.881 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:37.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:37 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/3237814219' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:51:38.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:38 smithi064 ceph-mon[104516]: pgmap v1664: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:38.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:38 smithi064 ceph-mon[109246]: pgmap v1664: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:38.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:38 smithi175 ceph-mon[101434]: pgmap v1664: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:38.960 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2199023255574 got 2199023255577 for osd.7 2024-08-17T17:51:38.960 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:39.839 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435654 got 468151435657 for osd.3 2024-08-17T17:51:39.839 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:40.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:40 smithi064 ceph-mon[104516]: pgmap v1665: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:40.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:40 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:40.780 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:40 smithi064 ceph-mon[109246]: pgmap v1665: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:40.780 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:40 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:40 smithi175 ceph-mon[101434]: pgmap v1665: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:40.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:40 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:41.956 INFO:teuthology.orchestra.run.smithi064.stdout:1060856922355 2024-08-17T17:51:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:42 smithi064 ceph-mon[104516]: pgmap v1666: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:42.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:42 smithi064 ceph-mon[104516]: from='client.? 172.21.15.64:0/1306153260' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:51:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:42 smithi064 ceph-mon[109246]: pgmap v1666: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:42.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:42 smithi064 ceph-mon[109246]: from='client.? 172.21.15.64:0/1306153260' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:51:42.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:42 smithi175 ceph-mon[101434]: pgmap v1666: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:42.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:42 smithi175 ceph-mon[101434]: from='client.? 172.21.15.64:0/1306153260' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:51:43.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1060856922351 got 1060856922355 for osd.6 2024-08-17T17:51:43.212 DEBUG:teuthology.parallel:result is None 2024-08-17T17:51:43.212 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:51:43.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:51:43.435 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[104516]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:51:43.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:51:43.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-mon[109246]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.981 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.981 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:43.981 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:51:43.982 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:51:43.982 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:43 smithi175 ceph-mon[101434]: from='mgr.25736 172.21.15.64:0/1256221702' entity='mgr.y' 2024-08-17T17:51:44.187 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:43 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: ::ffff:172.21.15.175 - - [17/Aug/2024:17:51:43] "GET /metrics HTTP/1.1" 200 34359 "" "Prometheus/2.51.0" 2024-08-17T17:51:44.886 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:44 smithi064 ceph-mon[104516]: pgmap v1667: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:44.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:44 smithi064 ceph-mon[109246]: pgmap v1667: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:45.160 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:44 smithi175 ceph-mon[101434]: pgmap v1667: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:51:45.331 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:51:46.121 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1668,"stamp":"2024-08-17T17:51:45.232050+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":203,"num_bytes_recovered":868993,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2672,"ondisk_log_size":2672,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6750424,"kb_used_data":68836,"kb_used_omap":54,"kb_used_meta":6681417,"kb_avail":743210792,"statfs":{"total":767960285184,"available":761047851008,"internally_reserved":0,"allocated":70488064,"data_stored":48882812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55424,"internal_metadata":6841771904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002701"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423510+0000","last_change":"2024-08-17T17:49:47.281246+0000","last_active":"2024-08-17T17:51:28.423510+0000","last_peered":"2024-08-17T17:51:28.423510+0000","last_clean":"2024-08-17T17:51:28.423510+0000","last_became_active":"2024-08-17T17:49:47.280793+0000","last_became_peered":"2024-08-17T17:49:47.280793+0000","last_unstale":"2024-08-17T17:51:28.423510+0000","last_undegraded":"2024-08-17T17:51:28.423510+0000","last_fullsized":"2024-08-17T17:51:28.423510+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:07.323680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":504,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419706+0000","last_change":"2024-08-17T17:49:38.180379+0000","last_active":"2024-08-17T17:51:27.419706+0000","last_peered":"2024-08-17T17:51:27.419706+0000","last_clean":"2024-08-17T17:51:27.419706+0000","last_became_active":"2024-08-17T17:49:38.178519+0000","last_became_peered":"2024-08-17T17:49:38.178519+0000","last_unstale":"2024-08-17T17:51:27.419706+0000","last_undegraded":"2024-08-17T17:51:27.419706+0000","last_fullsized":"2024-08-17T17:51:27.419706+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:52.254909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":364,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414649+0000","last_change":"2024-08-17T17:49:38.186598+0000","last_active":"2024-08-17T17:51:27.414649+0000","last_peered":"2024-08-17T17:51:27.414649+0000","last_clean":"2024-08-17T17:51:27.414649+0000","last_became_active":"2024-08-17T17:49:38.186221+0000","last_became_peered":"2024-08-17T17:49:38.186221+0000","last_unstale":"2024-08-17T17:51:27.414649+0000","last_undegraded":"2024-08-17T17:51:27.414649+0000","last_fullsized":"2024-08-17T17:51:27.414649+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:05:34.228626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":969,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.421535+0000","last_change":"2024-08-17T17:49:38.183188+0000","last_active":"2024-08-17T17:51:28.421535+0000","last_peered":"2024-08-17T17:51:28.421535+0000","last_clean":"2024-08-17T17:51:28.421535+0000","last_became_active":"2024-08-17T17:49:38.182753+0000","last_became_peered":"2024-08-17T17:49:38.182753+0000","last_unstale":"2024-08-17T17:51:28.421535+0000","last_undegraded":"2024-08-17T17:51:28.421535+0000","last_fullsized":"2024-08-17T17:51:28.421535+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:29.076167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":969,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422630+0000","last_change":"2024-08-17T17:49:38.183653+0000","last_active":"2024-08-17T17:51:28.422630+0000","last_peered":"2024-08-17T17:51:28.422630+0000","last_clean":"2024-08-17T17:51:28.422630+0000","last_became_active":"2024-08-17T17:49:38.182839+0000","last_became_peered":"2024-08-17T17:49:38.182839+0000","last_unstale":"2024-08-17T17:51:28.422630+0000","last_undegraded":"2024-08-17T17:51:28.422630+0000","last_fullsized":"2024-08-17T17:51:28.422630+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:04.137896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":493,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419397+0000","last_change":"2024-08-17T17:49:51.560582+0000","last_active":"2024-08-17T17:51:27.419397+0000","last_peered":"2024-08-17T17:51:27.419397+0000","last_clean":"2024-08-17T17:51:27.419397+0000","last_became_active":"2024-08-17T17:49:51.560137+0000","last_became_peered":"2024-08-17T17:49:51.560137+0000","last_unstale":"2024-08-17T17:51:27.419397+0000","last_undegraded":"2024-08-17T17:51:27.419397+0000","last_fullsized":"2024-08-17T17:51:27.419397+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:52.391000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":236,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.751388+0000","last_change":"2024-08-17T17:51:28.751318+0000","last_active":"2024-08-17T17:51:28.751388+0000","last_peered":"2024-08-17T17:51:28.751388+0000","last_clean":"2024-08-17T17:51:28.751388+0000","last_became_active":"2024-08-17T17:51:28.750953+0000","last_became_peered":"2024-08-17T17:51:28.750953+0000","last_unstale":"2024-08-17T17:51:28.751388+0000","last_undegraded":"2024-08-17T17:51:28.751388+0000","last_fullsized":"2024-08-17T17:51:28.751388+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:57.724249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":44,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.916887+0000","last_change":"2024-08-17T17:49:51.574488+0000","last_active":"2024-08-17T17:51:27.916887+0000","last_peered":"2024-08-17T17:51:27.916887+0000","last_clean":"2024-08-17T17:51:27.916887+0000","last_became_active":"2024-08-17T17:49:51.552176+0000","last_became_peered":"2024-08-17T17:49:51.552176+0000","last_unstale":"2024-08-17T17:51:27.916887+0000","last_undegraded":"2024-08-17T17:51:27.916887+0000","last_fullsized":"2024-08-17T17:51:27.916887+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:23.879437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":499,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.420035+0000","last_change":"2024-08-17T17:49:38.184627+0000","last_active":"2024-08-17T17:51:27.420035+0000","last_peered":"2024-08-17T17:51:27.420035+0000","last_clean":"2024-08-17T17:51:27.420035+0000","last_became_active":"2024-08-17T17:49:38.184204+0000","last_became_peered":"2024-08-17T17:49:38.184204+0000","last_unstale":"2024-08-17T17:51:27.420035+0000","last_undegraded":"2024-08-17T17:51:27.420035+0000","last_fullsized":"2024-08-17T17:51:27.420035+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:05.814301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":377,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414243+0000","last_change":"2024-08-17T17:49:51.557029+0000","last_active":"2024-08-17T17:51:27.414243+0000","last_peered":"2024-08-17T17:51:27.414243+0000","last_clean":"2024-08-17T17:51:27.414243+0000","last_became_active":"2024-08-17T17:49:51.556684+0000","last_became_peered":"2024-08-17T17:49:51.556684+0000","last_unstale":"2024-08-17T17:51:27.414243+0000","last_undegraded":"2024-08-17T17:51:27.414243+0000","last_fullsized":"2024-08-17T17:51:27.414243+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:45.942792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":511,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419836+0000","last_change":"2024-08-17T17:49:47.289138+0000","last_active":"2024-08-17T17:51:27.419836+0000","last_peered":"2024-08-17T17:51:27.419836+0000","last_clean":"2024-08-17T17:51:27.419836+0000","last_became_active":"2024-08-17T17:49:47.279224+0000","last_became_peered":"2024-08-17T17:49:47.279224+0000","last_unstale":"2024-08-17T17:51:27.419836+0000","last_undegraded":"2024-08-17T17:51:27.419836+0000","last_fullsized":"2024-08-17T17:51:27.419836+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:44.530583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.425578+0000","last_change":"2024-08-17T17:51:28.425578+0000","last_active":"2024-08-17T17:51:28.425578+0000","last_peered":"2024-08-17T17:51:28.425578+0000","last_clean":"2024-08-17T17:51:28.425578+0000","last_became_active":"2024-08-17T17:51:28.425253+0000","last_became_peered":"2024-08-17T17:51:28.425253+0000","last_unstale":"2024-08-17T17:51:28.425578+0000","last_undegraded":"2024-08-17T17:51:28.425578+0000","last_fullsized":"2024-08-17T17:51:28.425578+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:34.275222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":526,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419853+0000","last_change":"2024-08-17T17:49:51.569088+0000","last_active":"2024-08-17T17:51:27.419853+0000","last_peered":"2024-08-17T17:51:27.419853+0000","last_clean":"2024-08-17T17:51:27.419853+0000","last_became_active":"2024-08-17T17:49:51.560799+0000","last_became_peered":"2024-08-17T17:49:51.560799+0000","last_unstale":"2024-08-17T17:51:27.419853+0000","last_undegraded":"2024-08-17T17:51:27.419853+0000","last_fullsized":"2024-08-17T17:51:27.419853+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:41.005634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":34,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917139+0000","last_change":"2024-08-17T17:49:51.555421+0000","last_active":"2024-08-17T17:51:27.917139+0000","last_peered":"2024-08-17T17:51:27.917139+0000","last_clean":"2024-08-17T17:51:27.917139+0000","last_became_active":"2024-08-17T17:49:51.554855+0000","last_became_peered":"2024-08-17T17:49:51.554855+0000","last_unstale":"2024-08-17T17:51:27.917139+0000","last_undegraded":"2024-08-17T17:51:27.917139+0000","last_fullsized":"2024-08-17T17:51:27.917139+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:55.934318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":396,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.024387+0000","last_change":"2024-08-17T17:49:51.548401+0000","last_active":"2024-08-17T17:51:29.024387+0000","last_peered":"2024-08-17T17:51:29.024387+0000","last_clean":"2024-08-17T17:51:29.024387+0000","last_became_active":"2024-08-17T17:49:51.548026+0000","last_became_peered":"2024-08-17T17:49:51.548026+0000","last_unstale":"2024-08-17T17:51:29.024387+0000","last_undegraded":"2024-08-17T17:51:29.024387+0000","last_fullsized":"2024-08-17T17:51:29.024387+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:03.909757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":980,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422270+0000","last_change":"2024-08-17T17:49:47.277533+0000","last_active":"2024-08-17T17:51:28.422270+0000","last_peered":"2024-08-17T17:51:28.422270+0000","last_clean":"2024-08-17T17:51:28.422270+0000","last_became_active":"2024-08-17T17:49:47.277202+0000","last_became_peered":"2024-08-17T17:49:47.277202+0000","last_unstale":"2024-08-17T17:51:28.422270+0000","last_undegraded":"2024-08-17T17:51:28.422270+0000","last_fullsized":"2024-08-17T17:51:28.422270+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:36.468044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":897,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419528+0000","last_change":"2024-08-17T17:49:47.204803+0000","last_active":"2024-08-17T17:51:27.419528+0000","last_peered":"2024-08-17T17:51:27.419528+0000","last_clean":"2024-08-17T17:51:27.419528+0000","last_became_active":"2024-08-17T17:49:38.188991+0000","last_became_peered":"2024-08-17T17:49:38.188991+0000","last_unstale":"2024-08-17T17:51:27.419528+0000","last_undegraded":"2024-08-17T17:51:27.419528+0000","last_fullsized":"2024-08-17T17:51:27.419528+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:47.204650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:47.204650+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:10.937050+0000","scrub_duration":0.0018937310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":211,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423336+0000","last_change":"2024-08-17T17:47:08.509122+0000","last_active":"2024-08-17T17:51:28.423336+0000","last_peered":"2024-08-17T17:51:28.423336+0000","last_clean":"2024-08-17T17:51:28.423336+0000","last_became_active":"2024-08-17T17:47:08.497368+0000","last_became_peered":"2024-08-17T17:47:08.497368+0000","last_unstale":"2024-08-17T17:51:28.423336+0000","last_undegraded":"2024-08-17T17:51:28.423336+0000","last_fullsized":"2024-08-17T17:51:28.423336+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:35:01.638171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":529,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423988+0000","last_change":"2024-08-17T17:49:51.549530+0000","last_active":"2024-08-17T17:51:28.423988+0000","last_peered":"2024-08-17T17:51:28.423988+0000","last_clean":"2024-08-17T17:51:28.423988+0000","last_became_active":"2024-08-17T17:49:51.549190+0000","last_became_peered":"2024-08-17T17:49:51.549190+0000","last_unstale":"2024-08-17T17:51:28.423988+0000","last_undegraded":"2024-08-17T17:51:28.423988+0000","last_fullsized":"2024-08-17T17:51:28.423988+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:51.078221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":605,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414192+0000","last_change":"2024-08-17T17:47:11.534000+0000","last_active":"2024-08-17T17:51:27.414192+0000","last_peered":"2024-08-17T17:51:27.414192+0000","last_clean":"2024-08-17T17:51:27.414192+0000","last_became_active":"2024-08-17T17:47:11.533623+0000","last_became_peered":"2024-08-17T17:47:11.533623+0000","last_unstale":"2024-08-17T17:51:27.414192+0000","last_undegraded":"2024-08-17T17:51:27.414192+0000","last_fullsized":"2024-08-17T17:51:27.414192+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:28.627072+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":489,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917686+0000","last_change":"2024-08-17T17:50:51.098940+0000","last_active":"2024-08-17T17:51:27.917686+0000","last_peered":"2024-08-17T17:51:27.917686+0000","last_clean":"2024-08-17T17:51:27.917686+0000","last_became_active":"2024-08-17T17:50:51.098106+0000","last_became_peered":"2024-08-17T17:50:51.098106+0000","last_unstale":"2024-08-17T17:51:27.917686+0000","last_undegraded":"2024-08-17T17:51:27.917686+0000","last_fullsized":"2024-08-17T17:51:27.917686+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:26.596627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":458,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423980+0000","last_change":"2024-08-17T17:49:47.277724+0000","last_active":"2024-08-17T17:51:28.423980+0000","last_peered":"2024-08-17T17:51:28.423980+0000","last_clean":"2024-08-17T17:51:28.423980+0000","last_became_active":"2024-08-17T17:49:47.277400+0000","last_became_peered":"2024-08-17T17:49:47.277400+0000","last_unstale":"2024-08-17T17:51:28.423980+0000","last_undegraded":"2024-08-17T17:51:28.423980+0000","last_fullsized":"2024-08-17T17:51:28.423980+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:51.616464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":455,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.412727+0000","last_change":"2024-08-17T17:49:38.184894+0000","last_active":"2024-08-17T17:51:27.412727+0000","last_peered":"2024-08-17T17:51:27.412727+0000","last_clean":"2024-08-17T17:51:27.412727+0000","last_became_active":"2024-08-17T17:49:38.184532+0000","last_became_peered":"2024-08-17T17:49:38.184532+0000","last_unstale":"2024-08-17T17:51:27.412727+0000","last_undegraded":"2024-08-17T17:51:27.412727+0000","last_fullsized":"2024-08-17T17:51:27.412727+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:22.140571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":491,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423589+0000","last_change":"2024-08-17T17:49:51.559587+0000","last_active":"2024-08-17T17:51:28.423589+0000","last_peered":"2024-08-17T17:51:28.423589+0000","last_clean":"2024-08-17T17:51:28.423589+0000","last_became_active":"2024-08-17T17:49:51.557897+0000","last_became_peered":"2024-08-17T17:49:51.557897+0000","last_unstale":"2024-08-17T17:51:28.423589+0000","last_undegraded":"2024-08-17T17:51:28.423589+0000","last_fullsized":"2024-08-17T17:51:28.423589+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:11.838823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":597,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414258+0000","last_change":"2024-08-17T17:49:54.530915+0000","last_active":"2024-08-17T17:51:27.414258+0000","last_peered":"2024-08-17T17:51:27.414258+0000","last_clean":"2024-08-17T17:51:27.414258+0000","last_became_active":"2024-08-17T17:49:54.530492+0000","last_became_peered":"2024-08-17T17:49:54.530492+0000","last_unstale":"2024-08-17T17:51:27.414258+0000","last_undegraded":"2024-08-17T17:51:27.414258+0000","last_fullsized":"2024-08-17T17:51:27.414258+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:26.974624+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":230,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423228+0000","last_change":"2024-08-17T17:49:47.280381+0000","last_active":"2024-08-17T17:51:28.423228+0000","last_peered":"2024-08-17T17:51:28.423228+0000","last_clean":"2024-08-17T17:51:28.423228+0000","last_became_active":"2024-08-17T17:49:47.279935+0000","last_became_peered":"2024-08-17T17:49:47.279935+0000","last_unstale":"2024-08-17T17:51:28.423228+0000","last_undegraded":"2024-08-17T17:51:28.423228+0000","last_fullsized":"2024-08-17T17:51:28.423228+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:14:07.739243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031184+0000","last_change":"2024-08-17T17:49:47.286155+0000","last_active":"2024-08-17T17:51:29.031184+0000","last_peered":"2024-08-17T17:51:29.031184+0000","last_clean":"2024-08-17T17:51:29.031184+0000","last_became_active":"2024-08-17T17:49:47.285807+0000","last_became_peered":"2024-08-17T17:49:47.285807+0000","last_unstale":"2024-08-17T17:51:29.031184+0000","last_undegraded":"2024-08-17T17:51:29.031184+0000","last_fullsized":"2024-08-17T17:51:29.031184+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:01.217392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":872,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.749967+0000","last_change":"2024-08-17T17:49:54.529226+0000","last_active":"2024-08-17T17:51:28.749967+0000","last_peered":"2024-08-17T17:51:28.749967+0000","last_clean":"2024-08-17T17:51:28.749967+0000","last_became_active":"2024-08-17T17:49:54.528744+0000","last_became_peered":"2024-08-17T17:49:54.528744+0000","last_unstale":"2024-08-17T17:51:28.749967+0000","last_undegraded":"2024-08-17T17:51:28.749967+0000","last_fullsized":"2024-08-17T17:51:28.749967+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:13.216091+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:13.216091+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:59.992363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1155,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414405+0000","last_change":"2024-08-17T17:49:38.180140+0000","last_active":"2024-08-17T17:51:27.414405+0000","last_peered":"2024-08-17T17:51:27.414405+0000","last_clean":"2024-08-17T17:51:27.414405+0000","last_became_active":"2024-08-17T17:49:38.179484+0000","last_became_peered":"2024-08-17T17:49:38.179484+0000","last_unstale":"2024-08-17T17:51:27.414405+0000","last_undegraded":"2024-08-17T17:51:27.414405+0000","last_fullsized":"2024-08-17T17:51:27.414405+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:47.848809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":221,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423440+0000","last_change":"2024-08-17T17:49:51.548687+0000","last_active":"2024-08-17T17:51:28.423440+0000","last_peered":"2024-08-17T17:51:28.423440+0000","last_clean":"2024-08-17T17:51:28.423440+0000","last_became_active":"2024-08-17T17:49:51.548371+0000","last_became_peered":"2024-08-17T17:49:51.548371+0000","last_unstale":"2024-08-17T17:51:28.423440+0000","last_undegraded":"2024-08-17T17:51:28.423440+0000","last_fullsized":"2024-08-17T17:51:28.423440+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:02:26.441050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":472,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917527+0000","last_change":"2024-08-17T17:50:51.098517+0000","last_active":"2024-08-17T17:51:27.917527+0000","last_peered":"2024-08-17T17:51:27.917527+0000","last_clean":"2024-08-17T17:51:27.917527+0000","last_became_active":"2024-08-17T17:50:51.098175+0000","last_became_peered":"2024-08-17T17:50:51.098175+0000","last_unstale":"2024-08-17T17:51:27.917527+0000","last_undegraded":"2024-08-17T17:51:27.917527+0000","last_fullsized":"2024-08-17T17:51:27.917527+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:21.855681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":454,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030962+0000","last_change":"2024-08-17T17:49:54.530501+0000","last_active":"2024-08-17T17:51:29.030962+0000","last_peered":"2024-08-17T17:51:29.030962+0000","last_clean":"2024-08-17T17:51:29.030962+0000","last_became_active":"2024-08-17T17:49:54.530254+0000","last_became_peered":"2024-08-17T17:49:54.530254+0000","last_unstale":"2024-08-17T17:51:29.030962+0000","last_undegraded":"2024-08-17T17:51:29.030962+0000","last_fullsized":"2024-08-17T17:51:29.030962+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:49.874601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:49.874601+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:57:35.155147+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":927,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.749873+0000","last_change":"2024-08-17T17:49:47.298307+0000","last_active":"2024-08-17T17:51:28.749873+0000","last_peered":"2024-08-17T17:51:28.749873+0000","last_clean":"2024-08-17T17:51:28.749873+0000","last_became_active":"2024-08-17T17:49:47.277205+0000","last_became_peered":"2024-08-17T17:49:47.277205+0000","last_unstale":"2024-08-17T17:51:28.749873+0000","last_undegraded":"2024-08-17T17:51:28.749873+0000","last_fullsized":"2024-08-17T17:51:28.749873+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:30.940817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":974,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.421540+0000","last_change":"2024-08-17T17:47:08.495040+0000","last_active":"2024-08-17T17:51:28.421540+0000","last_peered":"2024-08-17T17:51:28.421540+0000","last_clean":"2024-08-17T17:51:28.421540+0000","last_became_active":"2024-08-17T17:47:08.494475+0000","last_became_peered":"2024-08-17T17:47:08.494475+0000","last_unstale":"2024-08-17T17:51:28.421540+0000","last_undegraded":"2024-08-17T17:51:28.421540+0000","last_fullsized":"2024-08-17T17:51:28.421540+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:17.169290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":982,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.026008+0000","last_change":"2024-08-17T17:51:29.025944+0000","last_active":"2024-08-17T17:51:29.026008+0000","last_peered":"2024-08-17T17:51:29.026008+0000","last_clean":"2024-08-17T17:51:29.026008+0000","last_became_active":"2024-08-17T17:51:29.025546+0000","last_became_peered":"2024-08-17T17:51:29.025546+0000","last_unstale":"2024-08-17T17:51:29.026008+0000","last_undegraded":"2024-08-17T17:51:29.026008+0000","last_fullsized":"2024-08-17T17:51:29.026008+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:58.766931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":591,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.750002+0000","last_change":"2024-08-17T17:49:38.183401+0000","last_active":"2024-08-17T17:51:28.750002+0000","last_peered":"2024-08-17T17:51:28.750002+0000","last_clean":"2024-08-17T17:51:28.750002+0000","last_became_active":"2024-08-17T17:49:38.181307+0000","last_became_peered":"2024-08-17T17:49:38.181307+0000","last_unstale":"2024-08-17T17:51:28.750002+0000","last_undegraded":"2024-08-17T17:51:28.750002+0000","last_fullsized":"2024-08-17T17:51:28.750002+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:54:28.351560+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":1056,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420212+0000","last_change":"2024-08-17T17:50:26.125178+0000","last_active":"2024-08-17T17:51:28.420212+0000","last_peered":"2024-08-17T17:51:28.420212+0000","last_clean":"2024-08-17T17:51:28.420212+0000","last_became_active":"2024-08-17T17:50:26.124567+0000","last_became_peered":"2024-08-17T17:50:26.124567+0000","last_unstale":"2024-08-17T17:51:28.420212+0000","last_undegraded":"2024-08-17T17:51:28.420212+0000","last_fullsized":"2024-08-17T17:51:28.420212+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:49.806769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030773+0000","last_change":"2024-08-17T17:49:51.556206+0000","last_active":"2024-08-17T17:51:29.030773+0000","last_peered":"2024-08-17T17:51:29.030773+0000","last_clean":"2024-08-17T17:51:29.030773+0000","last_became_active":"2024-08-17T17:49:51.555615+0000","last_became_peered":"2024-08-17T17:49:51.555615+0000","last_unstale":"2024-08-17T17:51:29.030773+0000","last_undegraded":"2024-08-17T17:51:29.030773+0000","last_fullsized":"2024-08-17T17:51:29.030773+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:11.845995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031213+0000","last_change":"2024-08-17T17:49:47.284436+0000","last_active":"2024-08-17T17:51:29.031213+0000","last_peered":"2024-08-17T17:51:29.031213+0000","last_clean":"2024-08-17T17:51:29.031213+0000","last_became_active":"2024-08-17T17:49:47.284114+0000","last_became_peered":"2024-08-17T17:49:47.284114+0000","last_unstale":"2024-08-17T17:51:29.031213+0000","last_undegraded":"2024-08-17T17:51:29.031213+0000","last_fullsized":"2024-08-17T17:51:29.031213+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:01.997142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":563,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414223+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:51:27.414223+0000","last_peered":"2024-08-17T17:51:27.414223+0000","last_clean":"2024-08-17T17:51:27.414223+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:51:27.414223+0000","last_undegraded":"2024-08-17T17:51:27.414223+0000","last_fullsized":"2024-08-17T17:51:27.414223+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1419,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414384+0000","last_change":"2024-08-17T17:49:54.530084+0000","last_active":"2024-08-17T17:51:27.414384+0000","last_peered":"2024-08-17T17:51:27.414384+0000","last_clean":"2024-08-17T17:51:27.414384+0000","last_became_active":"2024-08-17T17:49:54.529376+0000","last_became_peered":"2024-08-17T17:49:54.529376+0000","last_unstale":"2024-08-17T17:51:27.414384+0000","last_undegraded":"2024-08-17T17:51:27.414384+0000","last_fullsized":"2024-08-17T17:51:27.414384+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:35.411317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":803,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420086+0000","last_change":"2024-08-17T17:49:51.554056+0000","last_active":"2024-08-17T17:51:28.420086+0000","last_peered":"2024-08-17T17:51:28.420086+0000","last_clean":"2024-08-17T17:51:28.420086+0000","last_became_active":"2024-08-17T17:49:51.553672+0000","last_became_peered":"2024-08-17T17:49:51.553672+0000","last_unstale":"2024-08-17T17:51:28.420086+0000","last_undegraded":"2024-08-17T17:51:28.420086+0000","last_fullsized":"2024-08-17T17:51:28.420086+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:39.468612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030877+0000","last_change":"2024-08-17T17:49:47.279213+0000","last_active":"2024-08-17T17:51:29.030877+0000","last_peered":"2024-08-17T17:51:29.030877+0000","last_clean":"2024-08-17T17:51:29.030877+0000","last_became_active":"2024-08-17T17:49:47.277385+0000","last_became_peered":"2024-08-17T17:49:47.277385+0000","last_unstale":"2024-08-17T17:51:29.030877+0000","last_undegraded":"2024-08-17T17:51:29.030877+0000","last_fullsized":"2024-08-17T17:51:29.030877+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:22.617293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":902,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917728+0000","last_change":"2024-08-17T17:50:50.960461+0000","last_active":"2024-08-17T17:51:27.917728+0000","last_peered":"2024-08-17T17:51:27.917728+0000","last_clean":"2024-08-17T17:51:27.917728+0000","last_became_active":"2024-08-17T17:50:50.959945+0000","last_became_peered":"2024-08-17T17:50:50.959945+0000","last_unstale":"2024-08-17T17:51:27.917728+0000","last_undegraded":"2024-08-17T17:51:27.917728+0000","last_fullsized":"2024-08-17T17:51:27.917728+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:47:35.107980+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.107980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:13:40.869529+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":350,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414583+0000","last_change":"2024-08-17T17:47:08.492880+0000","last_active":"2024-08-17T17:51:27.414583+0000","last_peered":"2024-08-17T17:51:27.414583+0000","last_clean":"2024-08-17T17:51:27.414583+0000","last_became_active":"2024-08-17T17:47:08.492557+0000","last_became_peered":"2024-08-17T17:47:08.492557+0000","last_unstale":"2024-08-17T17:51:27.414583+0000","last_undegraded":"2024-08-17T17:51:27.414583+0000","last_fullsized":"2024-08-17T17:51:27.414583+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:33.714108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":432,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031058+0000","last_change":"2024-08-17T17:49:49.532036+0000","last_active":"2024-08-17T17:51:29.031058+0000","last_peered":"2024-08-17T17:51:29.031058+0000","last_clean":"2024-08-17T17:51:29.031058+0000","last_became_active":"2024-08-17T17:49:49.531215+0000","last_became_peered":"2024-08-17T17:49:49.531215+0000","last_unstale":"2024-08-17T17:51:29.031058+0000","last_undegraded":"2024-08-17T17:51:29.031058+0000","last_fullsized":"2024-08-17T17:51:29.031058+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:03.199364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030801+0000","last_change":"2024-08-17T17:49:47.278566+0000","last_active":"2024-08-17T17:51:29.030801+0000","last_peered":"2024-08-17T17:51:29.030801+0000","last_clean":"2024-08-17T17:51:29.030801+0000","last_became_active":"2024-08-17T17:49:47.277420+0000","last_became_peered":"2024-08-17T17:49:47.277420+0000","last_unstale":"2024-08-17T17:51:29.030801+0000","last_undegraded":"2024-08-17T17:51:29.030801+0000","last_fullsized":"2024-08-17T17:51:29.030801+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:53.784055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":755,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917404+0000","last_change":"2024-08-17T17:50:50.960043+0000","last_active":"2024-08-17T17:51:27.917404+0000","last_peered":"2024-08-17T17:51:27.917404+0000","last_clean":"2024-08-17T17:51:27.917404+0000","last_became_active":"2024-08-17T17:50:50.959684+0000","last_became_peered":"2024-08-17T17:50:50.959684+0000","last_unstale":"2024-08-17T17:51:27.917404+0000","last_undegraded":"2024-08-17T17:51:27.917404+0000","last_fullsized":"2024-08-17T17:51:27.917404+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:24:57.862662+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031068+0000","last_change":"2024-08-17T17:49:47.279083+0000","last_active":"2024-08-17T17:51:29.031068+0000","last_peered":"2024-08-17T17:51:29.031068+0000","last_clean":"2024-08-17T17:51:29.031068+0000","last_became_active":"2024-08-17T17:49:47.278261+0000","last_became_peered":"2024-08-17T17:49:47.278261+0000","last_unstale":"2024-08-17T17:51:29.031068+0000","last_undegraded":"2024-08-17T17:51:29.031068+0000","last_fullsized":"2024-08-17T17:51:29.031068+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:19.181493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1069,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.750326+0000","last_change":"2024-08-17T17:49:38.187221+0000","last_active":"2024-08-17T17:51:28.750326+0000","last_peered":"2024-08-17T17:51:28.750326+0000","last_clean":"2024-08-17T17:51:28.750326+0000","last_became_active":"2024-08-17T17:49:38.185868+0000","last_became_peered":"2024-08-17T17:49:38.185868+0000","last_unstale":"2024-08-17T17:51:28.750326+0000","last_undegraded":"2024-08-17T17:51:28.750326+0000","last_fullsized":"2024-08-17T17:51:28.750326+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:17:25.676554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":971,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422384+0000","last_change":"2024-08-17T17:49:38.179991+0000","last_active":"2024-08-17T17:51:28.422384+0000","last_peered":"2024-08-17T17:51:28.422384+0000","last_clean":"2024-08-17T17:51:28.422384+0000","last_became_active":"2024-08-17T17:49:38.179692+0000","last_became_peered":"2024-08-17T17:49:38.179692+0000","last_unstale":"2024-08-17T17:51:28.422384+0000","last_undegraded":"2024-08-17T17:51:28.422384+0000","last_fullsized":"2024-08-17T17:51:28.422384+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:38.218250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":598,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413524+0000","last_change":"2024-08-17T17:49:54.532355+0000","last_active":"2024-08-17T17:51:27.413524+0000","last_peered":"2024-08-17T17:51:27.413524+0000","last_clean":"2024-08-17T17:51:27.413524+0000","last_became_active":"2024-08-17T17:49:54.531952+0000","last_became_peered":"2024-08-17T17:49:54.531952+0000","last_unstale":"2024-08-17T17:51:27.413524+0000","last_undegraded":"2024-08-17T17:51:27.413524+0000","last_fullsized":"2024-08-17T17:51:27.413524+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:18.257051+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":222,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423894+0000","last_change":"2024-08-17T17:49:38.185555+0000","last_active":"2024-08-17T17:51:28.423894+0000","last_peered":"2024-08-17T17:51:28.423894+0000","last_clean":"2024-08-17T17:51:28.423894+0000","last_became_active":"2024-08-17T17:49:38.185152+0000","last_became_peered":"2024-08-17T17:49:38.185152+0000","last_unstale":"2024-08-17T17:51:28.423894+0000","last_undegraded":"2024-08-17T17:51:28.423894+0000","last_fullsized":"2024-08-17T17:51:28.423894+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:04.713862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":686,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031076+0000","last_change":"2024-08-17T17:49:54.530702+0000","last_active":"2024-08-17T17:51:29.031076+0000","last_peered":"2024-08-17T17:51:29.031076+0000","last_clean":"2024-08-17T17:51:29.031076+0000","last_became_active":"2024-08-17T17:49:54.530107+0000","last_became_peered":"2024-08-17T17:49:54.530107+0000","last_unstale":"2024-08-17T17:51:29.031076+0000","last_undegraded":"2024-08-17T17:51:29.031076+0000","last_fullsized":"2024-08-17T17:51:29.031076+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:42.121281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":218,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423276+0000","last_change":"2024-08-17T17:49:38.185435+0000","last_active":"2024-08-17T17:51:28.423276+0000","last_peered":"2024-08-17T17:51:28.423276+0000","last_clean":"2024-08-17T17:51:28.423276+0000","last_became_active":"2024-08-17T17:49:38.185133+0000","last_became_peered":"2024-08-17T17:49:38.185133+0000","last_unstale":"2024-08-17T17:51:28.423276+0000","last_undegraded":"2024-08-17T17:51:28.423276+0000","last_fullsized":"2024-08-17T17:51:28.423276+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:53.539480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":648,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423613+0000","last_change":"2024-08-17T17:49:38.184635+0000","last_active":"2024-08-17T17:51:28.423613+0000","last_peered":"2024-08-17T17:51:28.423613+0000","last_clean":"2024-08-17T17:51:28.423613+0000","last_became_active":"2024-08-17T17:49:38.184274+0000","last_became_peered":"2024-08-17T17:49:38.184274+0000","last_unstale":"2024-08-17T17:51:28.423613+0000","last_undegraded":"2024-08-17T17:51:28.423613+0000","last_fullsized":"2024-08-17T17:51:28.423613+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:08.775060+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":998,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422485+0000","last_change":"2024-08-17T17:49:47.290646+0000","last_active":"2024-08-17T17:51:28.422485+0000","last_peered":"2024-08-17T17:51:28.422485+0000","last_clean":"2024-08-17T17:51:28.422485+0000","last_became_active":"2024-08-17T17:49:47.281183+0000","last_became_peered":"2024-08-17T17:49:47.281183+0000","last_unstale":"2024-08-17T17:51:28.422485+0000","last_undegraded":"2024-08-17T17:51:28.422485+0000","last_fullsized":"2024-08-17T17:51:28.422485+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:50.323248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":146,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917577+0000","last_change":"2024-08-17T17:50:50.960934+0000","last_active":"2024-08-17T17:51:27.917577+0000","last_peered":"2024-08-17T17:51:27.917577+0000","last_clean":"2024-08-17T17:51:27.917577+0000","last_became_active":"2024-08-17T17:50:50.960386+0000","last_became_peered":"2024-08-17T17:50:50.960386+0000","last_unstale":"2024-08-17T17:51:27.917577+0000","last_undegraded":"2024-08-17T17:51:27.917577+0000","last_fullsized":"2024-08-17T17:51:27.917577+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:51.561386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":510,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.424213+0000","last_change":"2024-08-17T17:49:51.551000+0000","last_active":"2024-08-17T17:51:28.424213+0000","last_peered":"2024-08-17T17:51:28.424213+0000","last_clean":"2024-08-17T17:51:28.424213+0000","last_became_active":"2024-08-17T17:49:51.550677+0000","last_became_peered":"2024-08-17T17:49:51.550677+0000","last_unstale":"2024-08-17T17:51:28.424213+0000","last_undegraded":"2024-08-17T17:51:28.424213+0000","last_fullsized":"2024-08-17T17:51:28.424213+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:05.648670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413343+0000","last_change":"2024-08-17T17:49:51.558128+0000","last_active":"2024-08-17T17:51:27.413343+0000","last_peered":"2024-08-17T17:51:27.413343+0000","last_clean":"2024-08-17T17:51:27.413343+0000","last_became_active":"2024-08-17T17:49:51.557763+0000","last_became_peered":"2024-08-17T17:49:51.557763+0000","last_unstale":"2024-08-17T17:51:27.413343+0000","last_undegraded":"2024-08-17T17:51:27.413343+0000","last_fullsized":"2024-08-17T17:51:27.413343+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:24.114110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":519,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413327+0000","last_change":"2024-08-17T17:49:51.567488+0000","last_active":"2024-08-17T17:51:27.413327+0000","last_peered":"2024-08-17T17:51:27.413327+0000","last_clean":"2024-08-17T17:51:27.413327+0000","last_became_active":"2024-08-17T17:49:51.557813+0000","last_became_peered":"2024-08-17T17:49:51.557813+0000","last_unstale":"2024-08-17T17:51:27.413327+0000","last_undegraded":"2024-08-17T17:51:27.413327+0000","last_fullsized":"2024-08-17T17:51:27.413327+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:36.515914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14268,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030914+0000","last_change":"2024-08-17T17:49:51.558183+0000","last_active":"2024-08-17T17:51:29.030914+0000","last_peered":"2024-08-17T17:51:29.030914+0000","last_clean":"2024-08-17T17:51:29.030914+0000","last_became_active":"2024-08-17T17:49:51.556281+0000","last_became_peered":"2024-08-17T17:49:51.556281+0000","last_unstale":"2024-08-17T17:51:29.030914+0000","last_undegraded":"2024-08-17T17:51:29.030914+0000","last_fullsized":"2024-08-17T17:51:29.030914+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.405320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":524,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419676+0000","last_change":"2024-08-17T17:49:38.188823+0000","last_active":"2024-08-17T17:51:27.419676+0000","last_peered":"2024-08-17T17:51:27.419676+0000","last_clean":"2024-08-17T17:51:27.419676+0000","last_became_active":"2024-08-17T17:49:38.187590+0000","last_became_peered":"2024-08-17T17:49:38.187590+0000","last_unstale":"2024-08-17T17:51:27.419676+0000","last_undegraded":"2024-08-17T17:51:27.419676+0000","last_fullsized":"2024-08-17T17:51:27.419676+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:58:22.742541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":366,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414560+0000","last_change":"2024-08-17T17:49:47.276760+0000","last_active":"2024-08-17T17:51:27.414560+0000","last_peered":"2024-08-17T17:51:27.414560+0000","last_clean":"2024-08-17T17:51:27.414560+0000","last_became_active":"2024-08-17T17:49:47.276376+0000","last_became_peered":"2024-08-17T17:49:47.276376+0000","last_unstale":"2024-08-17T17:51:27.414560+0000","last_undegraded":"2024-08-17T17:51:27.414560+0000","last_fullsized":"2024-08-17T17:51:27.414560+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:31.990845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423626+0000","last_change":"2024-08-17T17:49:51.554953+0000","last_active":"2024-08-17T17:51:28.423626+0000","last_peered":"2024-08-17T17:51:28.423626+0000","last_clean":"2024-08-17T17:51:28.423626+0000","last_became_active":"2024-08-17T17:49:51.554572+0000","last_became_peered":"2024-08-17T17:49:51.554572+0000","last_unstale":"2024-08-17T17:51:28.423626+0000","last_undegraded":"2024-08-17T17:51:28.423626+0000","last_fullsized":"2024-08-17T17:51:28.423626+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:21.626987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":987,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422450+0000","last_change":"2024-08-17T17:47:08.492750+0000","last_active":"2024-08-17T17:51:28.422450+0000","last_peered":"2024-08-17T17:51:28.422450+0000","last_clean":"2024-08-17T17:51:28.422450+0000","last_became_active":"2024-08-17T17:47:08.492450+0000","last_became_peered":"2024-08-17T17:47:08.492450+0000","last_unstale":"2024-08-17T17:51:28.422450+0000","last_undegraded":"2024-08-17T17:51:28.422450+0000","last_fullsized":"2024-08-17T17:51:28.422450+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:51.781922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":234,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423375+0000","last_change":"2024-08-17T17:49:51.554301+0000","last_active":"2024-08-17T17:51:28.423375+0000","last_peered":"2024-08-17T17:51:28.423375+0000","last_clean":"2024-08-17T17:51:28.423375+0000","last_became_active":"2024-08-17T17:49:51.553935+0000","last_became_peered":"2024-08-17T17:49:51.553935+0000","last_unstale":"2024-08-17T17:51:28.423375+0000","last_undegraded":"2024-08-17T17:51:28.423375+0000","last_fullsized":"2024-08-17T17:51:28.423375+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.099463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":361,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917743+0000","last_change":"2024-08-17T17:50:51.099024+0000","last_active":"2024-08-17T17:51:27.917743+0000","last_peered":"2024-08-17T17:51:27.917743+0000","last_clean":"2024-08-17T17:51:27.917743+0000","last_became_active":"2024-08-17T17:50:51.098066+0000","last_became_peered":"2024-08-17T17:50:51.098066+0000","last_unstale":"2024-08-17T17:51:27.917743+0000","last_undegraded":"2024-08-17T17:51:27.917743+0000","last_fullsized":"2024-08-17T17:51:27.917743+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:19.739275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423144+0000","last_change":"2024-08-17T17:49:47.277906+0000","last_active":"2024-08-17T17:51:28.423144+0000","last_peered":"2024-08-17T17:51:28.423144+0000","last_clean":"2024-08-17T17:51:28.423144+0000","last_became_active":"2024-08-17T17:49:47.277581+0000","last_became_peered":"2024-08-17T17:49:47.277581+0000","last_unstale":"2024-08-17T17:51:28.423144+0000","last_undegraded":"2024-08-17T17:51:28.423144+0000","last_fullsized":"2024-08-17T17:51:28.423144+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:09:23.794337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":972,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422648+0000","last_change":"2024-08-17T17:49:38.184495+0000","last_active":"2024-08-17T17:51:28.422648+0000","last_peered":"2024-08-17T17:51:28.422648+0000","last_clean":"2024-08-17T17:51:28.422648+0000","last_became_active":"2024-08-17T17:49:38.183764+0000","last_became_peered":"2024-08-17T17:49:38.183764+0000","last_unstale":"2024-08-17T17:51:28.422648+0000","last_undegraded":"2024-08-17T17:51:28.422648+0000","last_fullsized":"2024-08-17T17:51:28.422648+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:29.940177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030704+0000","last_change":"2024-08-17T17:49:47.283190+0000","last_active":"2024-08-17T17:51:29.030704+0000","last_peered":"2024-08-17T17:51:29.030704+0000","last_clean":"2024-08-17T17:51:29.030704+0000","last_became_active":"2024-08-17T17:49:47.282859+0000","last_became_peered":"2024-08-17T17:49:47.282859+0000","last_unstale":"2024-08-17T17:51:29.030704+0000","last_undegraded":"2024-08-17T17:51:29.030704+0000","last_fullsized":"2024-08-17T17:51:29.030704+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:42.459724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423712+0000","last_change":"2024-08-17T17:49:47.278086+0000","last_active":"2024-08-17T17:51:28.423712+0000","last_peered":"2024-08-17T17:51:28.423712+0000","last_clean":"2024-08-17T17:51:28.423712+0000","last_became_active":"2024-08-17T17:49:47.277732+0000","last_became_peered":"2024-08-17T17:49:47.277732+0000","last_unstale":"2024-08-17T17:51:28.423712+0000","last_undegraded":"2024-08-17T17:51:28.423712+0000","last_fullsized":"2024-08-17T17:51:28.423712+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:22.200380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":456,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413040+0000","last_change":"2024-08-17T17:49:47.278017+0000","last_active":"2024-08-17T17:51:27.413040+0000","last_peered":"2024-08-17T17:51:27.413040+0000","last_clean":"2024-08-17T17:51:27.413040+0000","last_became_active":"2024-08-17T17:49:47.277656+0000","last_became_peered":"2024-08-17T17:49:47.277656+0000","last_unstale":"2024-08-17T17:51:27.413040+0000","last_undegraded":"2024-08-17T17:51:27.413040+0000","last_fullsized":"2024-08-17T17:51:27.413040+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:09.809817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":327,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420410+0000","last_change":"2024-08-17T17:50:26.124656+0000","last_active":"2024-08-17T17:51:28.420410+0000","last_peered":"2024-08-17T17:51:28.420410+0000","last_clean":"2024-08-17T17:51:28.420410+0000","last_became_active":"2024-08-17T17:50:26.124368+0000","last_became_peered":"2024-08-17T17:50:26.124368+0000","last_unstale":"2024-08-17T17:51:28.420410+0000","last_undegraded":"2024-08-17T17:51:28.420410+0000","last_fullsized":"2024-08-17T17:51:28.420410+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:54.371405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":351,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422023+0000","last_change":"2024-08-17T17:49:38.186511+0000","last_active":"2024-08-17T17:51:28.422023+0000","last_peered":"2024-08-17T17:51:28.422023+0000","last_clean":"2024-08-17T17:51:28.422023+0000","last_became_active":"2024-08-17T17:49:38.186222+0000","last_became_peered":"2024-08-17T17:49:38.186222+0000","last_unstale":"2024-08-17T17:51:28.422023+0000","last_undegraded":"2024-08-17T17:51:28.422023+0000","last_fullsized":"2024-08-17T17:51:28.422023+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:49.737705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":474,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413311+0000","last_change":"2024-08-17T17:49:47.278767+0000","last_active":"2024-08-17T17:51:27.413311+0000","last_peered":"2024-08-17T17:51:27.413311+0000","last_clean":"2024-08-17T17:51:27.413311+0000","last_became_active":"2024-08-17T17:49:47.278439+0000","last_became_peered":"2024-08-17T17:49:47.278439+0000","last_unstale":"2024-08-17T17:51:27.413311+0000","last_undegraded":"2024-08-17T17:51:27.413311+0000","last_fullsized":"2024-08-17T17:51:27.413311+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:28.278716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":372,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414480+0000","last_change":"2024-08-17T17:49:47.287126+0000","last_active":"2024-08-17T17:51:27.414480+0000","last_peered":"2024-08-17T17:51:27.414480+0000","last_clean":"2024-08-17T17:51:27.414480+0000","last_became_active":"2024-08-17T17:49:47.276538+0000","last_became_peered":"2024-08-17T17:49:47.276538+0000","last_unstale":"2024-08-17T17:51:27.414480+0000","last_undegraded":"2024-08-17T17:51:27.414480+0000","last_fullsized":"2024-08-17T17:51:27.414480+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:15.723828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":456,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413863+0000","last_change":"2024-08-17T17:47:08.491957+0000","last_active":"2024-08-17T17:51:27.413863+0000","last_peered":"2024-08-17T17:51:27.413863+0000","last_clean":"2024-08-17T17:51:27.413863+0000","last_became_active":"2024-08-17T17:47:08.491740+0000","last_became_peered":"2024-08-17T17:47:08.491740+0000","last_unstale":"2024-08-17T17:51:27.413863+0000","last_undegraded":"2024-08-17T17:51:27.413863+0000","last_fullsized":"2024-08-17T17:51:27.413863+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:05.976193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":397,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414680+0000","last_change":"2024-08-17T17:49:51.562547+0000","last_active":"2024-08-17T17:51:27.414680+0000","last_peered":"2024-08-17T17:51:27.414680+0000","last_clean":"2024-08-17T17:51:27.414680+0000","last_became_active":"2024-08-17T17:49:51.554773+0000","last_became_peered":"2024-08-17T17:49:51.554773+0000","last_unstale":"2024-08-17T17:51:27.414680+0000","last_undegraded":"2024-08-17T17:51:27.414680+0000","last_fullsized":"2024-08-17T17:51:27.414680+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:22:11.377068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":206,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423503+0000","last_change":"2024-08-17T17:47:08.499009+0000","last_active":"2024-08-17T17:51:28.423503+0000","last_peered":"2024-08-17T17:51:28.423503+0000","last_clean":"2024-08-17T17:51:28.423503+0000","last_became_active":"2024-08-17T17:47:08.498614+0000","last_became_peered":"2024-08-17T17:47:08.498614+0000","last_unstale":"2024-08-17T17:51:28.423503+0000","last_undegraded":"2024-08-17T17:51:28.423503+0000","last_fullsized":"2024-08-17T17:51:28.423503+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:15.246709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":240,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423772+0000","last_change":"2024-08-17T17:49:51.557566+0000","last_active":"2024-08-17T17:51:28.423772+0000","last_peered":"2024-08-17T17:51:28.423772+0000","last_clean":"2024-08-17T17:51:28.423772+0000","last_became_active":"2024-08-17T17:49:51.557152+0000","last_became_peered":"2024-08-17T17:49:51.557152+0000","last_unstale":"2024-08-17T17:51:28.423772+0000","last_undegraded":"2024-08-17T17:51:28.423772+0000","last_fullsized":"2024-08-17T17:51:28.423772+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:33.124411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":490,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.412727+0000","last_change":"2024-08-17T17:49:51.555074+0000","last_active":"2024-08-17T17:51:27.412727+0000","last_peered":"2024-08-17T17:51:27.412727+0000","last_clean":"2024-08-17T17:51:27.412727+0000","last_became_active":"2024-08-17T17:49:51.554090+0000","last_became_peered":"2024-08-17T17:49:51.554090+0000","last_unstale":"2024-08-17T17:51:27.412727+0000","last_undegraded":"2024-08-17T17:51:27.412727+0000","last_fullsized":"2024-08-17T17:51:27.412727+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:41.589529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":991,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422344+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:51:28.422344+0000","last_peered":"2024-08-17T17:51:28.422344+0000","last_clean":"2024-08-17T17:51:28.422344+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:51:28.422344+0000","last_undegraded":"2024-08-17T17:51:28.422344+0000","last_fullsized":"2024-08-17T17:51:28.422344+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":15842,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35786,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6894,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":512,"seq":2199023255579,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116960,"kb_used_data":11280,"kb_used_omap":8,"kb_used_meta":105655,"kb_avail":93628192,"statfs":{"total":95995035648,"available":95875268608,"internally_reserved":0,"allocated":11550720,"data_stored":8840629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9128,"internal_metadata":108190808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922356,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":595728,"kb_used_data":6852,"kb_used_omap":7,"kb_used_meta":588856,"kb_avail":93149424,"statfs":{"total":95995035648,"available":95385010176,"internally_reserved":0,"allocated":7016448,"data_stored":4318449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":602989238},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085879,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":716312,"kb_used_data":7432,"kb_used_omap":7,"kb_used_meta":708856,"kb_avail":93028840,"statfs":{"total":95995035648,"available":95261532160,"internally_reserved":0,"allocated":7610368,"data_stored":4905550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":725868985},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986769,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012576,"kb_used_data":10960,"kb_used_omap":1,"kb_used_meta":1001598,"kb_avail":92732576,"statfs":{"total":95995035648,"available":94958157824,"internally_reserved":0,"allocated":11223040,"data_stored":8513725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1025636806},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435661,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1067848,"kb_used_data":7424,"kb_used_omap":5,"kb_used_meta":1060410,"kb_avail":92677304,"statfs":{"total":95995035648,"available":94901559296,"internally_reserved":0,"allocated":7602176,"data_stored":4904989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":1085859913},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599187,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089680,"kb_used_data":7104,"kb_used_omap":7,"kb_used_meta":1082552,"kb_avail":92655472,"statfs":{"total":95995035648,"available":94879203328,"internally_reserved":0,"allocated":7274496,"data_stored":4576658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":1108533421},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762714,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967300,"kb_used_data":6836,"kb_used_omap":7,"kb_used_meta":960440,"kb_avail":92777852,"statfs":{"total":95995035648,"available":95004520448,"internally_reserved":0,"allocated":7000064,"data_stored":4313100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":983490977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926240,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184020,"kb_used_data":10948,"kb_used_omap":7,"kb_used_meta":1173048,"kb_avail":92561132,"statfs":{"total":95995035648,"available":94782599168,"internally_reserved":0,"allocated":11210752,"data_stored":8509712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":1201201756},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1645,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5894,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:51:46.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:51:46.342 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:46 smithi064 ceph-mon[104516]: pgmap v1668: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:46 smithi064 ceph-mon[104516]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:46.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:46 smithi064 ceph-mon[109246]: pgmap v1668: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:46.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:46 smithi064 ceph-mon[109246]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:46 smithi175 ceph-mon[101434]: pgmap v1668: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:47.161 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:46 smithi175 ceph-mon[101434]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: 2024-08-17T17:51:48.146 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2024-08-17T17:51:48.848 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1669,"stamp":"2024-08-17T17:51:47.232498+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":203,"num_bytes_recovered":868993,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2672,"ondisk_log_size":2672,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6750424,"kb_used_data":68836,"kb_used_omap":54,"kb_used_meta":6681417,"kb_avail":743210792,"statfs":{"total":767960285184,"available":761047851008,"internally_reserved":0,"allocated":70488064,"data_stored":48882812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55424,"internal_metadata":6841771904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002664"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423510+0000","last_change":"2024-08-17T17:49:47.281246+0000","last_active":"2024-08-17T17:51:28.423510+0000","last_peered":"2024-08-17T17:51:28.423510+0000","last_clean":"2024-08-17T17:51:28.423510+0000","last_became_active":"2024-08-17T17:49:47.280793+0000","last_became_peered":"2024-08-17T17:49:47.280793+0000","last_unstale":"2024-08-17T17:51:28.423510+0000","last_undegraded":"2024-08-17T17:51:28.423510+0000","last_fullsized":"2024-08-17T17:51:28.423510+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:07.323680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":504,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419706+0000","last_change":"2024-08-17T17:49:38.180379+0000","last_active":"2024-08-17T17:51:27.419706+0000","last_peered":"2024-08-17T17:51:27.419706+0000","last_clean":"2024-08-17T17:51:27.419706+0000","last_became_active":"2024-08-17T17:49:38.178519+0000","last_became_peered":"2024-08-17T17:49:38.178519+0000","last_unstale":"2024-08-17T17:51:27.419706+0000","last_undegraded":"2024-08-17T17:51:27.419706+0000","last_fullsized":"2024-08-17T17:51:27.419706+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:30:52.254909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":364,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414649+0000","last_change":"2024-08-17T17:49:38.186598+0000","last_active":"2024-08-17T17:51:27.414649+0000","last_peered":"2024-08-17T17:51:27.414649+0000","last_clean":"2024-08-17T17:51:27.414649+0000","last_became_active":"2024-08-17T17:49:38.186221+0000","last_became_peered":"2024-08-17T17:49:38.186221+0000","last_unstale":"2024-08-17T17:51:27.414649+0000","last_undegraded":"2024-08-17T17:51:27.414649+0000","last_fullsized":"2024-08-17T17:51:27.414649+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:05:34.228626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":969,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.421535+0000","last_change":"2024-08-17T17:49:38.183188+0000","last_active":"2024-08-17T17:51:28.421535+0000","last_peered":"2024-08-17T17:51:28.421535+0000","last_clean":"2024-08-17T17:51:28.421535+0000","last_became_active":"2024-08-17T17:49:38.182753+0000","last_became_peered":"2024-08-17T17:49:38.182753+0000","last_unstale":"2024-08-17T17:51:28.421535+0000","last_undegraded":"2024-08-17T17:51:28.421535+0000","last_fullsized":"2024-08-17T17:51:28.421535+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:16:29.076167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":969,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422630+0000","last_change":"2024-08-17T17:49:38.183653+0000","last_active":"2024-08-17T17:51:28.422630+0000","last_peered":"2024-08-17T17:51:28.422630+0000","last_clean":"2024-08-17T17:51:28.422630+0000","last_became_active":"2024-08-17T17:49:38.182839+0000","last_became_peered":"2024-08-17T17:49:38.182839+0000","last_unstale":"2024-08-17T17:51:28.422630+0000","last_undegraded":"2024-08-17T17:51:28.422630+0000","last_fullsized":"2024-08-17T17:51:28.422630+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:04.137896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":493,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419397+0000","last_change":"2024-08-17T17:49:51.560582+0000","last_active":"2024-08-17T17:51:27.419397+0000","last_peered":"2024-08-17T17:51:27.419397+0000","last_clean":"2024-08-17T17:51:27.419397+0000","last_became_active":"2024-08-17T17:49:51.560137+0000","last_became_peered":"2024-08-17T17:49:51.560137+0000","last_unstale":"2024-08-17T17:51:27.419397+0000","last_undegraded":"2024-08-17T17:51:27.419397+0000","last_fullsized":"2024-08-17T17:51:27.419397+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:52.391000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":236,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.751388+0000","last_change":"2024-08-17T17:51:28.751318+0000","last_active":"2024-08-17T17:51:28.751388+0000","last_peered":"2024-08-17T17:51:28.751388+0000","last_clean":"2024-08-17T17:51:28.751388+0000","last_became_active":"2024-08-17T17:51:28.750953+0000","last_became_peered":"2024-08-17T17:51:28.750953+0000","last_unstale":"2024-08-17T17:51:28.751388+0000","last_undegraded":"2024-08-17T17:51:28.751388+0000","last_fullsized":"2024-08-17T17:51:28.751388+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:57.724249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":44,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.916887+0000","last_change":"2024-08-17T17:49:51.574488+0000","last_active":"2024-08-17T17:51:27.916887+0000","last_peered":"2024-08-17T17:51:27.916887+0000","last_clean":"2024-08-17T17:51:27.916887+0000","last_became_active":"2024-08-17T17:49:51.552176+0000","last_became_peered":"2024-08-17T17:49:51.552176+0000","last_unstale":"2024-08-17T17:51:27.916887+0000","last_undegraded":"2024-08-17T17:51:27.916887+0000","last_fullsized":"2024-08-17T17:51:27.916887+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:23.879437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":499,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.420035+0000","last_change":"2024-08-17T17:49:38.184627+0000","last_active":"2024-08-17T17:51:27.420035+0000","last_peered":"2024-08-17T17:51:27.420035+0000","last_clean":"2024-08-17T17:51:27.420035+0000","last_became_active":"2024-08-17T17:49:38.184204+0000","last_became_peered":"2024-08-17T17:49:38.184204+0000","last_unstale":"2024-08-17T17:51:27.420035+0000","last_undegraded":"2024-08-17T17:51:27.420035+0000","last_fullsized":"2024-08-17T17:51:27.420035+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:05.814301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":377,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414243+0000","last_change":"2024-08-17T17:49:51.557029+0000","last_active":"2024-08-17T17:51:27.414243+0000","last_peered":"2024-08-17T17:51:27.414243+0000","last_clean":"2024-08-17T17:51:27.414243+0000","last_became_active":"2024-08-17T17:49:51.556684+0000","last_became_peered":"2024-08-17T17:49:51.556684+0000","last_unstale":"2024-08-17T17:51:27.414243+0000","last_undegraded":"2024-08-17T17:51:27.414243+0000","last_fullsized":"2024-08-17T17:51:27.414243+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:45.942792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":511,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419836+0000","last_change":"2024-08-17T17:49:47.289138+0000","last_active":"2024-08-17T17:51:27.419836+0000","last_peered":"2024-08-17T17:51:27.419836+0000","last_clean":"2024-08-17T17:51:27.419836+0000","last_became_active":"2024-08-17T17:49:47.279224+0000","last_became_peered":"2024-08-17T17:49:47.279224+0000","last_unstale":"2024-08-17T17:51:27.419836+0000","last_undegraded":"2024-08-17T17:51:27.419836+0000","last_fullsized":"2024-08-17T17:51:27.419836+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:44.530583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.425578+0000","last_change":"2024-08-17T17:51:28.425578+0000","last_active":"2024-08-17T17:51:28.425578+0000","last_peered":"2024-08-17T17:51:28.425578+0000","last_clean":"2024-08-17T17:51:28.425578+0000","last_became_active":"2024-08-17T17:51:28.425253+0000","last_became_peered":"2024-08-17T17:51:28.425253+0000","last_unstale":"2024-08-17T17:51:28.425578+0000","last_undegraded":"2024-08-17T17:51:28.425578+0000","last_fullsized":"2024-08-17T17:51:28.425578+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:34.275222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":526,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419853+0000","last_change":"2024-08-17T17:49:51.569088+0000","last_active":"2024-08-17T17:51:27.419853+0000","last_peered":"2024-08-17T17:51:27.419853+0000","last_clean":"2024-08-17T17:51:27.419853+0000","last_became_active":"2024-08-17T17:49:51.560799+0000","last_became_peered":"2024-08-17T17:49:51.560799+0000","last_unstale":"2024-08-17T17:51:27.419853+0000","last_undegraded":"2024-08-17T17:51:27.419853+0000","last_fullsized":"2024-08-17T17:51:27.419853+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:41.005634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":34,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917139+0000","last_change":"2024-08-17T17:49:51.555421+0000","last_active":"2024-08-17T17:51:27.917139+0000","last_peered":"2024-08-17T17:51:27.917139+0000","last_clean":"2024-08-17T17:51:27.917139+0000","last_became_active":"2024-08-17T17:49:51.554855+0000","last_became_peered":"2024-08-17T17:49:51.554855+0000","last_unstale":"2024-08-17T17:51:27.917139+0000","last_undegraded":"2024-08-17T17:51:27.917139+0000","last_fullsized":"2024-08-17T17:51:27.917139+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:55.934318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":396,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.024387+0000","last_change":"2024-08-17T17:49:51.548401+0000","last_active":"2024-08-17T17:51:29.024387+0000","last_peered":"2024-08-17T17:51:29.024387+0000","last_clean":"2024-08-17T17:51:29.024387+0000","last_became_active":"2024-08-17T17:49:51.548026+0000","last_became_peered":"2024-08-17T17:49:51.548026+0000","last_unstale":"2024-08-17T17:51:29.024387+0000","last_undegraded":"2024-08-17T17:51:29.024387+0000","last_fullsized":"2024-08-17T17:51:29.024387+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:03.909757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":980,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422270+0000","last_change":"2024-08-17T17:49:47.277533+0000","last_active":"2024-08-17T17:51:28.422270+0000","last_peered":"2024-08-17T17:51:28.422270+0000","last_clean":"2024-08-17T17:51:28.422270+0000","last_became_active":"2024-08-17T17:49:47.277202+0000","last_became_peered":"2024-08-17T17:49:47.277202+0000","last_unstale":"2024-08-17T17:51:28.422270+0000","last_undegraded":"2024-08-17T17:51:28.422270+0000","last_fullsized":"2024-08-17T17:51:28.422270+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:36.468044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":898,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.750077+0000","last_change":"2024-08-17T17:49:47.204803+0000","last_active":"2024-08-17T17:51:28.750077+0000","last_peered":"2024-08-17T17:51:28.750077+0000","last_clean":"2024-08-17T17:51:28.750077+0000","last_became_active":"2024-08-17T17:49:38.188991+0000","last_became_peered":"2024-08-17T17:49:38.188991+0000","last_unstale":"2024-08-17T17:51:28.750077+0000","last_undegraded":"2024-08-17T17:51:28.750077+0000","last_fullsized":"2024-08-17T17:51:28.750077+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:47.204650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:47.204650+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:10.937050+0000","scrub_duration":0.0018937310000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":211,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423336+0000","last_change":"2024-08-17T17:47:08.509122+0000","last_active":"2024-08-17T17:51:28.423336+0000","last_peered":"2024-08-17T17:51:28.423336+0000","last_clean":"2024-08-17T17:51:28.423336+0000","last_became_active":"2024-08-17T17:47:08.497368+0000","last_became_peered":"2024-08-17T17:47:08.497368+0000","last_unstale":"2024-08-17T17:51:28.423336+0000","last_undegraded":"2024-08-17T17:51:28.423336+0000","last_fullsized":"2024-08-17T17:51:28.423336+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:35:01.638171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":529,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423988+0000","last_change":"2024-08-17T17:49:51.549530+0000","last_active":"2024-08-17T17:51:28.423988+0000","last_peered":"2024-08-17T17:51:28.423988+0000","last_clean":"2024-08-17T17:51:28.423988+0000","last_became_active":"2024-08-17T17:49:51.549190+0000","last_became_peered":"2024-08-17T17:49:51.549190+0000","last_unstale":"2024-08-17T17:51:28.423988+0000","last_undegraded":"2024-08-17T17:51:28.423988+0000","last_fullsized":"2024-08-17T17:51:28.423988+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:51.078221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":605,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414192+0000","last_change":"2024-08-17T17:47:11.534000+0000","last_active":"2024-08-17T17:51:27.414192+0000","last_peered":"2024-08-17T17:51:27.414192+0000","last_clean":"2024-08-17T17:51:27.414192+0000","last_became_active":"2024-08-17T17:47:11.533623+0000","last_became_peered":"2024-08-17T17:47:11.533623+0000","last_unstale":"2024-08-17T17:51:27.414192+0000","last_undegraded":"2024-08-17T17:51:27.414192+0000","last_fullsized":"2024-08-17T17:51:27.414192+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:26.249726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:26.249726+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:28.627072+0000","scrub_duration":0.0052399769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":489,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917686+0000","last_change":"2024-08-17T17:50:51.098940+0000","last_active":"2024-08-17T17:51:27.917686+0000","last_peered":"2024-08-17T17:51:27.917686+0000","last_clean":"2024-08-17T17:51:27.917686+0000","last_became_active":"2024-08-17T17:50:51.098106+0000","last_became_peered":"2024-08-17T17:50:51.098106+0000","last_unstale":"2024-08-17T17:51:27.917686+0000","last_undegraded":"2024-08-17T17:51:27.917686+0000","last_fullsized":"2024-08-17T17:51:27.917686+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:00:26.596627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":458,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423980+0000","last_change":"2024-08-17T17:49:47.277724+0000","last_active":"2024-08-17T17:51:28.423980+0000","last_peered":"2024-08-17T17:51:28.423980+0000","last_clean":"2024-08-17T17:51:28.423980+0000","last_became_active":"2024-08-17T17:49:47.277400+0000","last_became_peered":"2024-08-17T17:49:47.277400+0000","last_unstale":"2024-08-17T17:51:28.423980+0000","last_undegraded":"2024-08-17T17:51:28.423980+0000","last_fullsized":"2024-08-17T17:51:28.423980+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:51.616464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":455,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.412727+0000","last_change":"2024-08-17T17:49:38.184894+0000","last_active":"2024-08-17T17:51:27.412727+0000","last_peered":"2024-08-17T17:51:27.412727+0000","last_clean":"2024-08-17T17:51:27.412727+0000","last_became_active":"2024-08-17T17:49:38.184532+0000","last_became_peered":"2024-08-17T17:49:38.184532+0000","last_unstale":"2024-08-17T17:51:27.412727+0000","last_undegraded":"2024-08-17T17:51:27.412727+0000","last_fullsized":"2024-08-17T17:51:27.412727+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:22.140571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":491,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423589+0000","last_change":"2024-08-17T17:49:51.559587+0000","last_active":"2024-08-17T17:51:28.423589+0000","last_peered":"2024-08-17T17:51:28.423589+0000","last_clean":"2024-08-17T17:51:28.423589+0000","last_became_active":"2024-08-17T17:49:51.557897+0000","last_became_peered":"2024-08-17T17:49:51.557897+0000","last_unstale":"2024-08-17T17:51:28.423589+0000","last_undegraded":"2024-08-17T17:51:28.423589+0000","last_fullsized":"2024-08-17T17:51:28.423589+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:11.838823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":597,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414258+0000","last_change":"2024-08-17T17:49:54.530915+0000","last_active":"2024-08-17T17:51:27.414258+0000","last_peered":"2024-08-17T17:51:27.414258+0000","last_clean":"2024-08-17T17:51:27.414258+0000","last_became_active":"2024-08-17T17:49:54.530492+0000","last_became_peered":"2024-08-17T17:49:54.530492+0000","last_unstale":"2024-08-17T17:51:27.414258+0000","last_undegraded":"2024-08-17T17:51:27.414258+0000","last_fullsized":"2024-08-17T17:51:27.414258+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:22.121637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:22.121637+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:26.974624+0000","scrub_duration":0.0027989260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":230,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423228+0000","last_change":"2024-08-17T17:49:47.280381+0000","last_active":"2024-08-17T17:51:28.423228+0000","last_peered":"2024-08-17T17:51:28.423228+0000","last_clean":"2024-08-17T17:51:28.423228+0000","last_became_active":"2024-08-17T17:49:47.279935+0000","last_became_peered":"2024-08-17T17:49:47.279935+0000","last_unstale":"2024-08-17T17:51:28.423228+0000","last_undegraded":"2024-08-17T17:51:28.423228+0000","last_fullsized":"2024-08-17T17:51:28.423228+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:14:07.739243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031184+0000","last_change":"2024-08-17T17:49:47.286155+0000","last_active":"2024-08-17T17:51:29.031184+0000","last_peered":"2024-08-17T17:51:29.031184+0000","last_clean":"2024-08-17T17:51:29.031184+0000","last_became_active":"2024-08-17T17:49:47.285807+0000","last_became_peered":"2024-08-17T17:49:47.285807+0000","last_unstale":"2024-08-17T17:51:29.031184+0000","last_undegraded":"2024-08-17T17:51:29.031184+0000","last_fullsized":"2024-08-17T17:51:29.031184+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:01.217392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":872,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.749967+0000","last_change":"2024-08-17T17:49:54.529226+0000","last_active":"2024-08-17T17:51:28.749967+0000","last_peered":"2024-08-17T17:51:28.749967+0000","last_clean":"2024-08-17T17:51:28.749967+0000","last_became_active":"2024-08-17T17:49:54.528744+0000","last_became_peered":"2024-08-17T17:49:54.528744+0000","last_unstale":"2024-08-17T17:51:28.749967+0000","last_undegraded":"2024-08-17T17:51:28.749967+0000","last_fullsized":"2024-08-17T17:51:28.749967+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:13.216091+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:13.216091+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:59.992363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'199","reported_seq":1155,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414405+0000","last_change":"2024-08-17T17:49:38.180140+0000","last_active":"2024-08-17T17:51:27.414405+0000","last_peered":"2024-08-17T17:51:27.414405+0000","last_clean":"2024-08-17T17:51:27.414405+0000","last_became_active":"2024-08-17T17:49:38.179484+0000","last_became_peered":"2024-08-17T17:49:38.179484+0000","last_unstale":"2024-08-17T17:51:27.414405+0000","last_undegraded":"2024-08-17T17:51:27.414405+0000","last_fullsized":"2024-08-17T17:51:27.414405+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:47.848809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056839399999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":828,"num_read_kb":882,"num_write":180,"num_write_kb":204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":221,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423440+0000","last_change":"2024-08-17T17:49:51.548687+0000","last_active":"2024-08-17T17:51:28.423440+0000","last_peered":"2024-08-17T17:51:28.423440+0000","last_clean":"2024-08-17T17:51:28.423440+0000","last_became_active":"2024-08-17T17:49:51.548371+0000","last_became_peered":"2024-08-17T17:49:51.548371+0000","last_unstale":"2024-08-17T17:51:28.423440+0000","last_undegraded":"2024-08-17T17:51:28.423440+0000","last_fullsized":"2024-08-17T17:51:28.423440+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:02:26.441050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":472,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917527+0000","last_change":"2024-08-17T17:50:51.098517+0000","last_active":"2024-08-17T17:51:27.917527+0000","last_peered":"2024-08-17T17:51:27.917527+0000","last_clean":"2024-08-17T17:51:27.917527+0000","last_became_active":"2024-08-17T17:50:51.098175+0000","last_became_peered":"2024-08-17T17:50:51.098175+0000","last_unstale":"2024-08-17T17:51:27.917527+0000","last_undegraded":"2024-08-17T17:51:27.917527+0000","last_fullsized":"2024-08-17T17:51:27.917527+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:21.855681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":454,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030962+0000","last_change":"2024-08-17T17:49:54.530501+0000","last_active":"2024-08-17T17:51:29.030962+0000","last_peered":"2024-08-17T17:51:29.030962+0000","last_clean":"2024-08-17T17:51:29.030962+0000","last_became_active":"2024-08-17T17:49:54.530254+0000","last_became_peered":"2024-08-17T17:49:54.530254+0000","last_unstale":"2024-08-17T17:51:29.030962+0000","last_undegraded":"2024-08-17T17:51:29.030962+0000","last_fullsized":"2024-08-17T17:51:29.030962+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:49:49.874601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:49:49.874601+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:57:35.155147+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":927,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.749873+0000","last_change":"2024-08-17T17:49:47.298307+0000","last_active":"2024-08-17T17:51:28.749873+0000","last_peered":"2024-08-17T17:51:28.749873+0000","last_clean":"2024-08-17T17:51:28.749873+0000","last_became_active":"2024-08-17T17:49:47.277205+0000","last_became_peered":"2024-08-17T17:49:47.277205+0000","last_unstale":"2024-08-17T17:51:28.749873+0000","last_undegraded":"2024-08-17T17:51:28.749873+0000","last_fullsized":"2024-08-17T17:51:28.749873+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:30.940817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044829400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":651,"num_read_kb":3063,"num_write":63,"num_write_kb":2295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":974,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.421540+0000","last_change":"2024-08-17T17:47:08.495040+0000","last_active":"2024-08-17T17:51:28.421540+0000","last_peered":"2024-08-17T17:51:28.421540+0000","last_clean":"2024-08-17T17:51:28.421540+0000","last_became_active":"2024-08-17T17:47:08.494475+0000","last_became_peered":"2024-08-17T17:47:08.494475+0000","last_unstale":"2024-08-17T17:51:28.421540+0000","last_undegraded":"2024-08-17T17:51:28.421540+0000","last_fullsized":"2024-08-17T17:51:28.421540+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:17.169290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":982,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.026008+0000","last_change":"2024-08-17T17:51:29.025944+0000","last_active":"2024-08-17T17:51:29.026008+0000","last_peered":"2024-08-17T17:51:29.026008+0000","last_clean":"2024-08-17T17:51:29.026008+0000","last_became_active":"2024-08-17T17:51:29.025546+0000","last_became_peered":"2024-08-17T17:51:29.025546+0000","last_unstale":"2024-08-17T17:51:29.026008+0000","last_undegraded":"2024-08-17T17:51:29.026008+0000","last_fullsized":"2024-08-17T17:51:29.026008+0000","mapping_epoch":543,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:34:58.766931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":591,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.750002+0000","last_change":"2024-08-17T17:49:38.183401+0000","last_active":"2024-08-17T17:51:28.750002+0000","last_peered":"2024-08-17T17:51:28.750002+0000","last_clean":"2024-08-17T17:51:28.750002+0000","last_became_active":"2024-08-17T17:49:38.181307+0000","last_became_peered":"2024-08-17T17:49:38.181307+0000","last_unstale":"2024-08-17T17:51:28.750002+0000","last_undegraded":"2024-08-17T17:51:28.750002+0000","last_fullsized":"2024-08-17T17:51:28.750002+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:21.029122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:21.029122+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:54:28.351560+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'84","reported_seq":1056,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420212+0000","last_change":"2024-08-17T17:50:26.125178+0000","last_active":"2024-08-17T17:51:28.420212+0000","last_peered":"2024-08-17T17:51:28.420212+0000","last_clean":"2024-08-17T17:51:28.420212+0000","last_became_active":"2024-08-17T17:50:26.124567+0000","last_became_peered":"2024-08-17T17:50:26.124567+0000","last_unstale":"2024-08-17T17:51:28.420212+0000","last_undegraded":"2024-08-17T17:51:28.420212+0000","last_fullsized":"2024-08-17T17:51:28.420212+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:49.806769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063212000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":621,"num_read_kb":1612,"num_write":67,"num_write_kb":1124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030773+0000","last_change":"2024-08-17T17:49:51.556206+0000","last_active":"2024-08-17T17:51:29.030773+0000","last_peered":"2024-08-17T17:51:29.030773+0000","last_clean":"2024-08-17T17:51:29.030773+0000","last_became_active":"2024-08-17T17:49:51.555615+0000","last_became_peered":"2024-08-17T17:49:51.555615+0000","last_unstale":"2024-08-17T17:51:29.030773+0000","last_undegraded":"2024-08-17T17:51:29.030773+0000","last_fullsized":"2024-08-17T17:51:29.030773+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:11.845995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031213+0000","last_change":"2024-08-17T17:49:47.284436+0000","last_active":"2024-08-17T17:51:29.031213+0000","last_peered":"2024-08-17T17:51:29.031213+0000","last_clean":"2024-08-17T17:51:29.031213+0000","last_became_active":"2024-08-17T17:49:47.284114+0000","last_became_peered":"2024-08-17T17:49:47.284114+0000","last_unstale":"2024-08-17T17:51:29.031213+0000","last_undegraded":"2024-08-17T17:51:29.031213+0000","last_fullsized":"2024-08-17T17:51:29.031213+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:01.997142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":563,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414223+0000","last_change":"2024-08-17T17:40:19.147732+0000","last_active":"2024-08-17T17:51:27.414223+0000","last_peered":"2024-08-17T17:51:27.414223+0000","last_clean":"2024-08-17T17:51:27.414223+0000","last_became_active":"2024-08-17T17:39:41.345243+0000","last_became_peered":"2024-08-17T17:39:41.345243+0000","last_unstale":"2024-08-17T17:51:27.414223+0000","last_undegraded":"2024-08-17T17:51:27.414223+0000","last_fullsized":"2024-08-17T17:51:27.414223+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:19.147596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:19.147596+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:24.868158+0000","scrub_duration":0.0043726609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'147","reported_seq":1419,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414384+0000","last_change":"2024-08-17T17:49:54.530084+0000","last_active":"2024-08-17T17:51:27.414384+0000","last_peered":"2024-08-17T17:51:27.414384+0000","last_clean":"2024-08-17T17:51:27.414384+0000","last_became_active":"2024-08-17T17:49:54.529376+0000","last_became_peered":"2024-08-17T17:49:54.529376+0000","last_unstale":"2024-08-17T17:51:27.414384+0000","last_undegraded":"2024-08-17T17:51:27.414384+0000","last_fullsized":"2024-08-17T17:51:27.414384+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":147,"log_dups_size":0,"ondisk_log_size":147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:35.411317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055761299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1448,"num_read_kb":1454,"num_write":106,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":803,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420086+0000","last_change":"2024-08-17T17:49:51.554056+0000","last_active":"2024-08-17T17:51:28.420086+0000","last_peered":"2024-08-17T17:51:28.420086+0000","last_clean":"2024-08-17T17:51:28.420086+0000","last_became_active":"2024-08-17T17:49:51.553672+0000","last_became_peered":"2024-08-17T17:49:51.553672+0000","last_unstale":"2024-08-17T17:51:28.420086+0000","last_undegraded":"2024-08-17T17:51:28.420086+0000","last_fullsized":"2024-08-17T17:51:28.420086+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:39.468612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030877+0000","last_change":"2024-08-17T17:49:47.279213+0000","last_active":"2024-08-17T17:51:29.030877+0000","last_peered":"2024-08-17T17:51:29.030877+0000","last_clean":"2024-08-17T17:51:29.030877+0000","last_became_active":"2024-08-17T17:49:47.277385+0000","last_became_peered":"2024-08-17T17:49:47.277385+0000","last_unstale":"2024-08-17T17:51:29.030877+0000","last_undegraded":"2024-08-17T17:51:29.030877+0000","last_fullsized":"2024-08-17T17:51:29.030877+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:22.617293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":902,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917728+0000","last_change":"2024-08-17T17:50:50.960461+0000","last_active":"2024-08-17T17:51:27.917728+0000","last_peered":"2024-08-17T17:51:27.917728+0000","last_clean":"2024-08-17T17:51:27.917728+0000","last_became_active":"2024-08-17T17:50:50.959945+0000","last_became_peered":"2024-08-17T17:50:50.959945+0000","last_unstale":"2024-08-17T17:51:27.917728+0000","last_undegraded":"2024-08-17T17:51:27.917728+0000","last_fullsized":"2024-08-17T17:51:27.917728+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:47:35.107980+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-17T17:44:52.091249+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.107980+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:13:40.869529+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":350,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414583+0000","last_change":"2024-08-17T17:47:08.492880+0000","last_active":"2024-08-17T17:51:27.414583+0000","last_peered":"2024-08-17T17:51:27.414583+0000","last_clean":"2024-08-17T17:51:27.414583+0000","last_became_active":"2024-08-17T17:47:08.492557+0000","last_became_peered":"2024-08-17T17:47:08.492557+0000","last_unstale":"2024-08-17T17:51:27.414583+0000","last_undegraded":"2024-08-17T17:51:27.414583+0000","last_fullsized":"2024-08-17T17:51:27.414583+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:33.714108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'113","reported_seq":432,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031058+0000","last_change":"2024-08-17T17:49:49.532036+0000","last_active":"2024-08-17T17:51:29.031058+0000","last_peered":"2024-08-17T17:51:29.031058+0000","last_clean":"2024-08-17T17:51:29.031058+0000","last_became_active":"2024-08-17T17:49:49.531215+0000","last_became_peered":"2024-08-17T17:49:49.531215+0000","last_unstale":"2024-08-17T17:51:29.031058+0000","last_undegraded":"2024-08-17T17:51:29.031058+0000","last_fullsized":"2024-08-17T17:51:29.031058+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:03.199364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056845799999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":329,"num_read_kb":1282,"num_write":139,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030801+0000","last_change":"2024-08-17T17:49:47.278566+0000","last_active":"2024-08-17T17:51:29.030801+0000","last_peered":"2024-08-17T17:51:29.030801+0000","last_clean":"2024-08-17T17:51:29.030801+0000","last_became_active":"2024-08-17T17:49:47.277420+0000","last_became_peered":"2024-08-17T17:49:47.277420+0000","last_unstale":"2024-08-17T17:51:29.030801+0000","last_undegraded":"2024-08-17T17:51:29.030801+0000","last_fullsized":"2024-08-17T17:51:29.030801+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:53.784055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":755,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917404+0000","last_change":"2024-08-17T17:50:50.960043+0000","last_active":"2024-08-17T17:51:27.917404+0000","last_peered":"2024-08-17T17:51:27.917404+0000","last_clean":"2024-08-17T17:51:27.917404+0000","last_became_active":"2024-08-17T17:50:50.959684+0000","last_became_peered":"2024-08-17T17:50:50.959684+0000","last_unstale":"2024-08-17T17:51:27.917404+0000","last_undegraded":"2024-08-17T17:51:27.917404+0000","last_fullsized":"2024-08-17T17:51:27.917404+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.020636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.020636+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:24:57.862662+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031068+0000","last_change":"2024-08-17T17:49:47.279083+0000","last_active":"2024-08-17T17:51:29.031068+0000","last_peered":"2024-08-17T17:51:29.031068+0000","last_clean":"2024-08-17T17:51:29.031068+0000","last_became_active":"2024-08-17T17:49:47.278261+0000","last_became_peered":"2024-08-17T17:49:47.278261+0000","last_unstale":"2024-08-17T17:51:29.031068+0000","last_undegraded":"2024-08-17T17:51:29.031068+0000","last_fullsized":"2024-08-17T17:51:29.031068+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:19.181493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'131","reported_seq":1069,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.750326+0000","last_change":"2024-08-17T17:49:38.187221+0000","last_active":"2024-08-17T17:51:28.750326+0000","last_peered":"2024-08-17T17:51:28.750326+0000","last_clean":"2024-08-17T17:51:28.750326+0000","last_became_active":"2024-08-17T17:49:38.185868+0000","last_became_peered":"2024-08-17T17:49:38.185868+0000","last_unstale":"2024-08-17T17:51:28.750326+0000","last_undegraded":"2024-08-17T17:51:28.750326+0000","last_fullsized":"2024-08-17T17:51:28.750326+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:17:25.676554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088644799999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":966,"num_read_kb":2428,"num_write":122,"num_write_kb":5480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":971,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422384+0000","last_change":"2024-08-17T17:49:38.179991+0000","last_active":"2024-08-17T17:51:28.422384+0000","last_peered":"2024-08-17T17:51:28.422384+0000","last_clean":"2024-08-17T17:51:28.422384+0000","last_became_active":"2024-08-17T17:49:38.179692+0000","last_became_peered":"2024-08-17T17:49:38.179692+0000","last_unstale":"2024-08-17T17:51:28.422384+0000","last_undegraded":"2024-08-17T17:51:28.422384+0000","last_fullsized":"2024-08-17T17:51:28.422384+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:38.218250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":598,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413524+0000","last_change":"2024-08-17T17:49:54.532355+0000","last_active":"2024-08-17T17:51:27.413524+0000","last_peered":"2024-08-17T17:51:27.413524+0000","last_clean":"2024-08-17T17:51:27.413524+0000","last_became_active":"2024-08-17T17:49:54.531952+0000","last_became_peered":"2024-08-17T17:49:54.531952+0000","last_unstale":"2024-08-17T17:51:27.413524+0000","last_undegraded":"2024-08-17T17:51:27.413524+0000","last_fullsized":"2024-08-17T17:51:27.413524+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:40:20.537734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:40:20.537734+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:18.257051+0000","scrub_duration":0.0021053410000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":222,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423894+0000","last_change":"2024-08-17T17:49:38.185555+0000","last_active":"2024-08-17T17:51:28.423894+0000","last_peered":"2024-08-17T17:51:28.423894+0000","last_clean":"2024-08-17T17:51:28.423894+0000","last_became_active":"2024-08-17T17:49:38.185152+0000","last_became_peered":"2024-08-17T17:49:38.185152+0000","last_unstale":"2024-08-17T17:51:28.423894+0000","last_undegraded":"2024-08-17T17:51:28.423894+0000","last_fullsized":"2024-08-17T17:51:28.423894+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:04.713862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'148","reported_seq":686,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.031076+0000","last_change":"2024-08-17T17:49:54.530702+0000","last_active":"2024-08-17T17:51:29.031076+0000","last_peered":"2024-08-17T17:51:29.031076+0000","last_clean":"2024-08-17T17:51:29.031076+0000","last_became_active":"2024-08-17T17:49:54.530107+0000","last_became_peered":"2024-08-17T17:49:54.530107+0000","last_unstale":"2024-08-17T17:51:29.031076+0000","last_undegraded":"2024-08-17T17:51:29.031076+0000","last_fullsized":"2024-08-17T17:51:29.031076+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:42.121281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023829100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":1594,"num_write":116,"num_write_kb":1071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":218,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423276+0000","last_change":"2024-08-17T17:49:38.185435+0000","last_active":"2024-08-17T17:51:28.423276+0000","last_peered":"2024-08-17T17:51:28.423276+0000","last_clean":"2024-08-17T17:51:28.423276+0000","last_became_active":"2024-08-17T17:49:38.185133+0000","last_became_peered":"2024-08-17T17:49:38.185133+0000","last_unstale":"2024-08-17T17:51:28.423276+0000","last_undegraded":"2024-08-17T17:51:28.423276+0000","last_fullsized":"2024-08-17T17:51:28.423276+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:53.539480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":648,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423613+0000","last_change":"2024-08-17T17:49:38.184635+0000","last_active":"2024-08-17T17:51:28.423613+0000","last_peered":"2024-08-17T17:51:28.423613+0000","last_clean":"2024-08-17T17:51:28.423613+0000","last_became_active":"2024-08-17T17:49:38.184274+0000","last_became_peered":"2024-08-17T17:49:38.184274+0000","last_unstale":"2024-08-17T17:51:28.423613+0000","last_undegraded":"2024-08-17T17:51:28.423613+0000","last_fullsized":"2024-08-17T17:51:28.423613+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":367,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-17T17:44:51.111900+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:01:48.140299+0000","last_clean_scrub_stamp":"2024-08-17T17:44:51.111900+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:08.775060+0000","scrub_duration":0.001767942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":998,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422485+0000","last_change":"2024-08-17T17:49:47.290646+0000","last_active":"2024-08-17T17:51:28.422485+0000","last_peered":"2024-08-17T17:51:28.422485+0000","last_clean":"2024-08-17T17:51:28.422485+0000","last_became_active":"2024-08-17T17:49:47.281183+0000","last_became_peered":"2024-08-17T17:49:47.281183+0000","last_unstale":"2024-08-17T17:51:28.422485+0000","last_undegraded":"2024-08-17T17:51:28.422485+0000","last_fullsized":"2024-08-17T17:51:28.422485+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:50.323248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'69","reported_seq":146,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917577+0000","last_change":"2024-08-17T17:50:50.960934+0000","last_active":"2024-08-17T17:51:27.917577+0000","last_peered":"2024-08-17T17:51:27.917577+0000","last_clean":"2024-08-17T17:51:27.917577+0000","last_became_active":"2024-08-17T17:50:50.960386+0000","last_became_peered":"2024-08-17T17:50:50.960386+0000","last_unstale":"2024-08-17T17:51:27.917577+0000","last_undegraded":"2024-08-17T17:51:27.917577+0000","last_fullsized":"2024-08-17T17:51:27.917577+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:04:23.292352+0000","last_clean_scrub_stamp":"2024-08-17T17:04:23.292352+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:07:51.561386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025994800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":684,"num_write":53,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":510,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.424213+0000","last_change":"2024-08-17T17:49:51.551000+0000","last_active":"2024-08-17T17:51:28.424213+0000","last_peered":"2024-08-17T17:51:28.424213+0000","last_clean":"2024-08-17T17:51:28.424213+0000","last_became_active":"2024-08-17T17:49:51.550677+0000","last_became_peered":"2024-08-17T17:49:51.550677+0000","last_unstale":"2024-08-17T17:51:28.424213+0000","last_undegraded":"2024-08-17T17:51:28.424213+0000","last_fullsized":"2024-08-17T17:51:28.424213+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:05.648670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413343+0000","last_change":"2024-08-17T17:49:51.558128+0000","last_active":"2024-08-17T17:51:27.413343+0000","last_peered":"2024-08-17T17:51:27.413343+0000","last_clean":"2024-08-17T17:51:27.413343+0000","last_became_active":"2024-08-17T17:49:51.557763+0000","last_became_peered":"2024-08-17T17:49:51.557763+0000","last_unstale":"2024-08-17T17:51:27.413343+0000","last_undegraded":"2024-08-17T17:51:27.413343+0000","last_fullsized":"2024-08-17T17:51:27.413343+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:24.114110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":519,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413327+0000","last_change":"2024-08-17T17:49:51.567488+0000","last_active":"2024-08-17T17:51:27.413327+0000","last_peered":"2024-08-17T17:51:27.413327+0000","last_clean":"2024-08-17T17:51:27.413327+0000","last_became_active":"2024-08-17T17:49:51.557813+0000","last_became_peered":"2024-08-17T17:49:51.557813+0000","last_unstale":"2024-08-17T17:51:27.413327+0000","last_undegraded":"2024-08-17T17:51:27.413327+0000","last_fullsized":"2024-08-17T17:51:27.413327+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:36.515914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14268,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030914+0000","last_change":"2024-08-17T17:49:51.558183+0000","last_active":"2024-08-17T17:51:29.030914+0000","last_peered":"2024-08-17T17:51:29.030914+0000","last_clean":"2024-08-17T17:51:29.030914+0000","last_became_active":"2024-08-17T17:49:51.556281+0000","last_became_peered":"2024-08-17T17:49:51.556281+0000","last_unstale":"2024-08-17T17:51:29.030914+0000","last_undegraded":"2024-08-17T17:51:29.030914+0000","last_fullsized":"2024-08-17T17:51:29.030914+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:25:28.405320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":524,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.419676+0000","last_change":"2024-08-17T17:49:38.188823+0000","last_active":"2024-08-17T17:51:27.419676+0000","last_peered":"2024-08-17T17:51:27.419676+0000","last_clean":"2024-08-17T17:51:27.419676+0000","last_became_active":"2024-08-17T17:49:38.187590+0000","last_became_peered":"2024-08-17T17:49:38.187590+0000","last_unstale":"2024-08-17T17:51:27.419676+0000","last_undegraded":"2024-08-17T17:51:27.419676+0000","last_fullsized":"2024-08-17T17:51:27.419676+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:58:22.742541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":366,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414560+0000","last_change":"2024-08-17T17:49:47.276760+0000","last_active":"2024-08-17T17:51:27.414560+0000","last_peered":"2024-08-17T17:51:27.414560+0000","last_clean":"2024-08-17T17:51:27.414560+0000","last_became_active":"2024-08-17T17:49:47.276376+0000","last_became_peered":"2024-08-17T17:49:47.276376+0000","last_unstale":"2024-08-17T17:51:27.414560+0000","last_undegraded":"2024-08-17T17:51:27.414560+0000","last_fullsized":"2024-08-17T17:51:27.414560+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:31.990845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":229,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423626+0000","last_change":"2024-08-17T17:49:51.554953+0000","last_active":"2024-08-17T17:51:28.423626+0000","last_peered":"2024-08-17T17:51:28.423626+0000","last_clean":"2024-08-17T17:51:28.423626+0000","last_became_active":"2024-08-17T17:49:51.554572+0000","last_became_peered":"2024-08-17T17:49:51.554572+0000","last_unstale":"2024-08-17T17:51:28.423626+0000","last_undegraded":"2024-08-17T17:51:28.423626+0000","last_fullsized":"2024-08-17T17:51:28.423626+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:21.626987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":987,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422450+0000","last_change":"2024-08-17T17:47:08.492750+0000","last_active":"2024-08-17T17:51:28.422450+0000","last_peered":"2024-08-17T17:51:28.422450+0000","last_clean":"2024-08-17T17:51:28.422450+0000","last_became_active":"2024-08-17T17:47:08.492450+0000","last_became_peered":"2024-08-17T17:47:08.492450+0000","last_unstale":"2024-08-17T17:51:28.422450+0000","last_undegraded":"2024-08-17T17:51:28.422450+0000","last_fullsized":"2024-08-17T17:51:28.422450+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:51.781922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":234,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423375+0000","last_change":"2024-08-17T17:49:51.554301+0000","last_active":"2024-08-17T17:51:28.423375+0000","last_peered":"2024-08-17T17:51:28.423375+0000","last_clean":"2024-08-17T17:51:28.423375+0000","last_became_active":"2024-08-17T17:49:51.553935+0000","last_became_peered":"2024-08-17T17:49:51.553935+0000","last_unstale":"2024-08-17T17:51:28.423375+0000","last_undegraded":"2024-08-17T17:51:28.423375+0000","last_fullsized":"2024-08-17T17:51:28.423375+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.099463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":361,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.917743+0000","last_change":"2024-08-17T17:50:51.099024+0000","last_active":"2024-08-17T17:51:27.917743+0000","last_peered":"2024-08-17T17:51:27.917743+0000","last_clean":"2024-08-17T17:51:27.917743+0000","last_became_active":"2024-08-17T17:50:51.098066+0000","last_became_peered":"2024-08-17T17:50:51.098066+0000","last_unstale":"2024-08-17T17:51:27.917743+0000","last_undegraded":"2024-08-17T17:51:27.917743+0000","last_fullsized":"2024-08-17T17:51:27.917743+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:19.739275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423144+0000","last_change":"2024-08-17T17:49:47.277906+0000","last_active":"2024-08-17T17:51:28.423144+0000","last_peered":"2024-08-17T17:51:28.423144+0000","last_clean":"2024-08-17T17:51:28.423144+0000","last_became_active":"2024-08-17T17:49:47.277581+0000","last_became_peered":"2024-08-17T17:49:47.277581+0000","last_unstale":"2024-08-17T17:51:28.423144+0000","last_undegraded":"2024-08-17T17:51:28.423144+0000","last_fullsized":"2024-08-17T17:51:28.423144+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:09:23.794337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":972,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422648+0000","last_change":"2024-08-17T17:49:38.184495+0000","last_active":"2024-08-17T17:51:28.422648+0000","last_peered":"2024-08-17T17:51:28.422648+0000","last_clean":"2024-08-17T17:51:28.422648+0000","last_became_active":"2024-08-17T17:49:38.183764+0000","last_became_peered":"2024-08-17T17:49:38.183764+0000","last_unstale":"2024-08-17T17:51:28.422648+0000","last_undegraded":"2024-08-17T17:51:28.422648+0000","last_fullsized":"2024-08-17T17:51:28.422648+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:29.940177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:29.030704+0000","last_change":"2024-08-17T17:49:47.283190+0000","last_active":"2024-08-17T17:51:29.030704+0000","last_peered":"2024-08-17T17:51:29.030704+0000","last_clean":"2024-08-17T17:51:29.030704+0000","last_became_active":"2024-08-17T17:49:47.282859+0000","last_became_peered":"2024-08-17T17:49:47.282859+0000","last_unstale":"2024-08-17T17:51:29.030704+0000","last_undegraded":"2024-08-17T17:51:29.030704+0000","last_fullsized":"2024-08-17T17:51:29.030704+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:42.459724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423712+0000","last_change":"2024-08-17T17:49:47.278086+0000","last_active":"2024-08-17T17:51:28.423712+0000","last_peered":"2024-08-17T17:51:28.423712+0000","last_clean":"2024-08-17T17:51:28.423712+0000","last_became_active":"2024-08-17T17:49:47.277732+0000","last_became_peered":"2024-08-17T17:49:47.277732+0000","last_unstale":"2024-08-17T17:51:28.423712+0000","last_undegraded":"2024-08-17T17:51:28.423712+0000","last_fullsized":"2024-08-17T17:51:28.423712+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:22.200380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":456,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413040+0000","last_change":"2024-08-17T17:49:47.278017+0000","last_active":"2024-08-17T17:51:27.413040+0000","last_peered":"2024-08-17T17:51:27.413040+0000","last_clean":"2024-08-17T17:51:27.413040+0000","last_became_active":"2024-08-17T17:49:47.277656+0000","last_became_peered":"2024-08-17T17:49:47.277656+0000","last_unstale":"2024-08-17T17:51:27.413040+0000","last_undegraded":"2024-08-17T17:51:27.413040+0000","last_fullsized":"2024-08-17T17:51:27.413040+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:09.809817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":327,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.420410+0000","last_change":"2024-08-17T17:50:26.124656+0000","last_active":"2024-08-17T17:51:28.420410+0000","last_peered":"2024-08-17T17:51:28.420410+0000","last_clean":"2024-08-17T17:51:28.420410+0000","last_became_active":"2024-08-17T17:50:26.124368+0000","last_became_peered":"2024-08-17T17:50:26.124368+0000","last_unstale":"2024-08-17T17:51:28.420410+0000","last_undegraded":"2024-08-17T17:51:28.420410+0000","last_fullsized":"2024-08-17T17:51:28.420410+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:54.371405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":351,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422023+0000","last_change":"2024-08-17T17:49:38.186511+0000","last_active":"2024-08-17T17:51:28.422023+0000","last_peered":"2024-08-17T17:51:28.422023+0000","last_clean":"2024-08-17T17:51:28.422023+0000","last_became_active":"2024-08-17T17:49:38.186222+0000","last_became_peered":"2024-08-17T17:49:38.186222+0000","last_unstale":"2024-08-17T17:51:28.422023+0000","last_undegraded":"2024-08-17T17:51:28.422023+0000","last_fullsized":"2024-08-17T17:51:28.422023+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:49.737705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":474,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413311+0000","last_change":"2024-08-17T17:49:47.278767+0000","last_active":"2024-08-17T17:51:27.413311+0000","last_peered":"2024-08-17T17:51:27.413311+0000","last_clean":"2024-08-17T17:51:27.413311+0000","last_became_active":"2024-08-17T17:49:47.278439+0000","last_became_peered":"2024-08-17T17:49:47.278439+0000","last_unstale":"2024-08-17T17:51:27.413311+0000","last_undegraded":"2024-08-17T17:51:27.413311+0000","last_fullsized":"2024-08-17T17:51:27.413311+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:28.278716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":372,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414480+0000","last_change":"2024-08-17T17:49:47.287126+0000","last_active":"2024-08-17T17:51:27.414480+0000","last_peered":"2024-08-17T17:51:27.414480+0000","last_clean":"2024-08-17T17:51:27.414480+0000","last_became_active":"2024-08-17T17:49:47.276538+0000","last_became_peered":"2024-08-17T17:49:47.276538+0000","last_unstale":"2024-08-17T17:51:27.414480+0000","last_undegraded":"2024-08-17T17:51:27.414480+0000","last_fullsized":"2024-08-17T17:51:27.414480+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:15.723828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":456,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.413863+0000","last_change":"2024-08-17T17:47:08.491957+0000","last_active":"2024-08-17T17:51:27.413863+0000","last_peered":"2024-08-17T17:51:27.413863+0000","last_clean":"2024-08-17T17:51:27.413863+0000","last_became_active":"2024-08-17T17:47:08.491740+0000","last_became_peered":"2024-08-17T17:47:08.491740+0000","last_unstale":"2024-08-17T17:51:27.413863+0000","last_undegraded":"2024-08-17T17:51:27.413863+0000","last_fullsized":"2024-08-17T17:51:27.413863+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:05.976193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":397,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.414680+0000","last_change":"2024-08-17T17:49:51.562547+0000","last_active":"2024-08-17T17:51:27.414680+0000","last_peered":"2024-08-17T17:51:27.414680+0000","last_clean":"2024-08-17T17:51:27.414680+0000","last_became_active":"2024-08-17T17:49:51.554773+0000","last_became_peered":"2024-08-17T17:49:51.554773+0000","last_unstale":"2024-08-17T17:51:27.414680+0000","last_undegraded":"2024-08-17T17:51:27.414680+0000","last_fullsized":"2024-08-17T17:51:27.414680+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:22:11.377068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":206,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423503+0000","last_change":"2024-08-17T17:47:08.499009+0000","last_active":"2024-08-17T17:51:28.423503+0000","last_peered":"2024-08-17T17:51:28.423503+0000","last_clean":"2024-08-17T17:51:28.423503+0000","last_became_active":"2024-08-17T17:47:08.498614+0000","last_became_peered":"2024-08-17T17:47:08.498614+0000","last_unstale":"2024-08-17T17:51:28.423503+0000","last_undegraded":"2024-08-17T17:51:28.423503+0000","last_fullsized":"2024-08-17T17:51:28.423503+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:15.246709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":240,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.423772+0000","last_change":"2024-08-17T17:49:51.557566+0000","last_active":"2024-08-17T17:51:28.423772+0000","last_peered":"2024-08-17T17:51:28.423772+0000","last_clean":"2024-08-17T17:51:28.423772+0000","last_became_active":"2024-08-17T17:49:51.557152+0000","last_became_peered":"2024-08-17T17:49:51.557152+0000","last_unstale":"2024-08-17T17:51:28.423772+0000","last_undegraded":"2024-08-17T17:51:28.423772+0000","last_fullsized":"2024-08-17T17:51:28.423772+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:03:33.124411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":490,"reported_epoch":543,"state":"active+clean","last_fresh":"2024-08-17T17:51:27.412727+0000","last_change":"2024-08-17T17:49:51.555074+0000","last_active":"2024-08-17T17:51:27.412727+0000","last_peered":"2024-08-17T17:51:27.412727+0000","last_clean":"2024-08-17T17:51:27.412727+0000","last_became_active":"2024-08-17T17:49:51.554090+0000","last_became_peered":"2024-08-17T17:49:51.554090+0000","last_unstale":"2024-08-17T17:51:27.412727+0000","last_undegraded":"2024-08-17T17:51:27.412727+0000","last_fullsized":"2024-08-17T17:51:27.412727+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:34.047642+0000","last_clean_scrub_stamp":"2024-08-17T17:05:34.047642+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:41.589529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":991,"reported_epoch":544,"state":"active+clean","last_fresh":"2024-08-17T17:51:28.422344+0000","last_change":"2024-08-17T17:31:17.595002+0000","last_active":"2024-08-17T17:51:28.422344+0000","last_peered":"2024-08-17T17:51:28.422344+0000","last_clean":"2024-08-17T17:51:28.422344+0000","last_became_active":"2024-08-17T17:31:17.594321+0000","last_became_peered":"2024-08-17T17:31:17.594321+0000","last_unstale":"2024-08-17T17:51:28.422344+0000","last_undegraded":"2024-08-17T17:51:28.422344+0000","last_fullsized":"2024-08-17T17:51:28.422344+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:05:33.039582+0000","last_clean_scrub_stamp":"2024-08-17T17:05:33.039582+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:54.084685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":15842,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35786,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5986,"num_read_kb":12999,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6894,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":512,"seq":2199023255580,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116960,"kb_used_data":11280,"kb_used_omap":8,"kb_used_meta":105655,"kb_avail":93628192,"statfs":{"total":95995035648,"available":95875268608,"internally_reserved":0,"allocated":11550720,"data_stored":8840629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9128,"internal_metadata":108190808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":247,"seq":1060856922356,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":595728,"kb_used_data":6852,"kb_used_omap":7,"kb_used_meta":588856,"kb_avail":93149424,"statfs":{"total":95995035648,"available":95385010176,"internally_reserved":0,"allocated":7016448,"data_stored":4318449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7498,"internal_metadata":602989238},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":242,"seq":1039382085880,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":716312,"kb_used_data":7432,"kb_used_omap":7,"kb_used_meta":708856,"kb_avail":93028840,"statfs":{"total":95995035648,"available":95261532160,"internally_reserved":0,"allocated":7610368,"data_stored":4905550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7751,"internal_metadata":725868985},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986770,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012576,"kb_used_data":10960,"kb_used_omap":1,"kb_used_meta":1001598,"kb_avail":92732576,"statfs":{"total":95995035648,"available":94958157824,"internally_reserved":0,"allocated":11223040,"data_stored":8513725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1025636806},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435661,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1067848,"kb_used_data":7424,"kb_used_omap":5,"kb_used_meta":1060410,"kb_avail":92677304,"statfs":{"total":95995035648,"available":94901559296,"internally_reserved":0,"allocated":7602176,"data_stored":4904989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6071,"internal_metadata":1085859913},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676599188,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089680,"kb_used_data":7104,"kb_used_omap":7,"kb_used_meta":1082552,"kb_avail":92655472,"statfs":{"total":95995035648,"available":94879203328,"internally_reserved":0,"allocated":7274496,"data_stored":4576658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8019,"internal_metadata":1108533421},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762714,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967300,"kb_used_data":6836,"kb_used_omap":7,"kb_used_meta":960440,"kb_avail":92777852,"statfs":{"total":95995035648,"available":95004520448,"internally_reserved":0,"allocated":7000064,"data_stored":4313100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7775,"internal_metadata":983490977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926240,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184020,"kb_used_data":10948,"kb_used_omap":7,"kb_used_meta":1173048,"kb_avail":92561132,"statfs":{"total":95995035648,"available":94782599168,"internally_reserved":0,"allocated":11210752,"data_stored":8509712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7588,"internal_metadata":1201201756},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1645,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5994,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6181,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2479,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3176,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5905,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5894,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:51:48.851 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:51:48.852 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:51:49.053 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=500ba2cef4d640c4ae6d6d52ae9b3c57 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:51:49.056 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-17T17:51:49.065 INFO:tasks.cephadm:Teardown begin 2024-08-17T17:51:49.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:51:49.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:48 smithi064 ceph-mon[104516]: pgmap v1669: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:49.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:48 smithi064 ceph-mon[109246]: pgmap v1669: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:49.127 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:51:49.153 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:48 smithi175 ceph-mon[101434]: pgmap v1669: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:51:49.173 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-17T17:51:49.173 DEBUG:teuthology.orchestra.run.smithi064:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:51:49.192 DEBUG:teuthology.orchestra.run.smithi175:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:51:49.236 INFO:tasks.cephadm:Stopping all daemons... 2024-08-17T17:51:49.237 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-17T17:51:49.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a 2024-08-17T17:51:49.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:49 smithi064 systemd[1]: Stopping Ceph mon.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:49.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:49] ENGINE Bus STOPPING 2024-08-17T17:51:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:51:49.547+0000 7fb081a03640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:51:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a[104512]: 2024-08-17T17:51:49.547+0000 7fb081a03640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-17T17:51:49.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:49 smithi064 podman[347699]: 2024-08-17 17:51:49.792814941 +0000 UTC m=+0.345547292 container died 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_CLEAN=True) 2024-08-17T17:51:50.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:51:50.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:49] ENGINE Bus STOPPED 2024-08-17T17:51:50.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:49 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:49] ENGINE Bus STARTING 2024-08-17T17:51:50.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:50 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:50] ENGINE Serving on http://:::9283 2024-08-17T17:51:50.290 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:50 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: [17/Aug/2024:17:51:50] ENGINE Bus STARTED 2024-08-17T17:51:50.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:50 smithi064 podman[347699]: 2024-08-17 17:51:50.289634458 +0000 UTC m=+0.842366795 container cleanup 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_CLEAN=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:51:50.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:50 smithi064 bash[347699]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:51:50.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Aug 17 17:51:50 smithi064 podman[347710]: 2024-08-17 17:51:50.50780048 +0000 UTC m=+0.713053197 container remove 1eec4cd20a357410cd6d74718dfef6f462b7ad84846568fa47e7274e339ad81e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-a, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux ) 2024-08-17T17:51:50.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.a.service' 2024-08-17T17:51:50.701 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:51:50.702 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-17T17:51:50.702 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-17T17:51:50.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.c 2024-08-17T17:51:50.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:50 smithi064 systemd[1]: Stopping Ceph mon.c for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:51.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c[109242]: 2024-08-17T17:51:51.002+0000 7f98b624c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:51:51.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:51 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c[109242]: 2024-08-17T17:51:51.002+0000 7f98b624c640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:51:51.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:51 smithi064 podman[347832]: 2024-08-17 17:51:51.064225552 +0000 UTC m=+0.173759333 container died 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:51:51.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:51 smithi064 podman[347832]: 2024-08-17 17:51:51.452860104 +0000 UTC m=+0.562393839 container cleanup 00207a16fa93d5e628f18f24dfcf256977d5ded51e5525d7575c2a4668022395 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:51:51.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Aug 17 17:51:51 smithi064 bash[347832]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:51:51.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.c.service' 2024-08-17T17:51:51.902 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:51:51.902 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-17T17:51:51.902 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-17T17:51:51.903 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b 2024-08-17T17:51:52.234 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:51 smithi175 systemd[1]: Stopping Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:52.651 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b[101430]: 2024-08-17T17:51:52.231+0000 7f761b9b6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:51:52.651 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b[101430]: 2024-08-17T17:51:52.231+0000 7f761b9b6640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:51:52.651 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:52 smithi175 podman[252435]: 2024-08-17 17:51:52.335865747 +0000 UTC m=+0.232533841 container died 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:51:52.910 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:52 smithi175 podman[252435]: 2024-08-17 17:51:52.849767264 +0000 UTC m=+0.746435345 container cleanup 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-17T17:51:52.911 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:52 smithi175 bash[252435]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:51:53.321 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service' 2024-08-17T17:51:53.344 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:53 smithi175 podman[252447]: 2024-08-17 17:51:53.092977576 +0000 UTC m=+0.751382071 container remove 9bdd73bb1f26df66695ee21a0debfbbce0063fd91f4cc37d9dc3eb73bd975699 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mon-b, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:51:53.345 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:53 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:51:53.345 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:53 smithi175 systemd[1]: Stopped Ceph mon.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:51:53.345 INFO:journalctl@ceph.mon.b.smithi175.stdout:Aug 17 17:51:53 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mon.b.service: Consumed 22.272s CPU time. 2024-08-17T17:51:53.359 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:51:53.360 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-17T17:51:53.360 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-17T17:51:53.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y 2024-08-17T17:51:53.677 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:53 smithi064 systemd[1]: Stopping Ceph mgr.y for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:53.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:53 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y[86909]: 2024-08-17T17:51:53.675+0000 7f06c6df6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:51:53.937 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:53 smithi064 podman[347952]: 2024-08-17 17:51:53.718956039 +0000 UTC m=+0.160923131 container died af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe) 2024-08-17T17:51:54.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:54 smithi064 podman[347952]: 2024-08-17 17:51:54.215697777 +0000 UTC m=+0.657664921 container cleanup af5009d72be5043c24522d2e484ea944e95763b16d4324ac4883b1e5421c3dc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe) 2024-08-17T17:51:54.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Aug 17 17:51:54 smithi064 bash[347952]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:51:54.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.y.service' 2024-08-17T17:51:54.635 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:51:54.636 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-17T17:51:54.636 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-17T17:51:54.636 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x 2024-08-17T17:51:55.011 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:51:54 smithi175 systemd[1]: Stopping Ceph mgr.x for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:55.316 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:51:55 smithi175 podman[252554]: 2024-08-17 17:51:55.010555807 +0000 UTC m=+0.168894025 container died 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T17:51:55.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:51:55 smithi175 podman[252554]: 2024-08-17 17:51:55.404657632 +0000 UTC m=+0.562995849 container cleanup 683d25124077409909bae4bcb4c6449c53dde0c7013b7f3c9a6b828baa8996bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-08-17T17:51:55.661 INFO:journalctl@ceph.mgr.x.smithi175.stdout:Aug 17 17:51:55 smithi175 bash[252554]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:51:55.827 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@mgr.x.service' 2024-08-17T17:51:55.861 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:51:55.861 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-17T17:51:55.861 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-17T17:51:55.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.0 2024-08-17T17:51:56.173 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:51:55 smithi064 systemd[1]: Stopping Ceph osd.0 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:51:56.437 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:51:56 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:51:56.172+0000 7f39b5063640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:51:56.437 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:51:56 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:51:56.172+0000 7f39b5063640 -1 osd.0 544 *** Got signal Terminated *** 2024-08-17T17:51:56.437 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:51:56 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0[120595]: 2024-08-17T17:51:56.172+0000 7f39b5063640 -1 osd.0 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:02.145 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:01 smithi064 podman[348072]: 2024-08-17 17:52:01.815459021 +0000 UTC m=+5.754177857 container died 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-08-17T17:52:02.437 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:02 smithi064 podman[348072]: 2024-08-17 17:52:02.315053774 +0000 UTC m=+6.253772610 container cleanup 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:02.437 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:02 smithi064 bash[348072]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:52:02.937 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:02 smithi064 podman[348087]: 2024-08-17 17:52:02.54157889 +0000 UTC m=+0.720643139 container remove 63f96c51dd267214a615d4009aed7db1310aac061fa7bcb8682aab20c8f3f1bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_CLEAN=True) 2024-08-17T17:52:03.229 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:02 smithi064 podman[348175]: 2024-08-17 17:52:02.874419843 +0000 UTC m=+0.017882309 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:52:03.230 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:02 smithi064 podman[348175]: 2024-08-17 17:52:02.984373876 +0000 UTC m=+0.127836334 container create cd3fe0cf64f18a013c21a2e7829418f5711fcd2b0a91f1abb446f591301ca4da (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:52:03.230 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:03 smithi064 podman[348175]: 2024-08-17 17:52:03.22912677 +0000 UTC m=+0.372589248 container init cd3fe0cf64f18a013c21a2e7829418f5711fcd2b0a91f1abb446f591301ca4da (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:52:03.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:03 smithi064 podman[348175]: 2024-08-17 17:52:03.232868648 +0000 UTC m=+0.376331107 container start cd3fe0cf64f18a013c21a2e7829418f5711fcd2b0a91f1abb446f591301ca4da (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:52:03.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:03 smithi064 podman[348175]: 2024-08-17 17:52:03.31185961 +0000 UTC m=+0.455322073 container attach cd3fe0cf64f18a013c21a2e7829418f5711fcd2b0a91f1abb446f591301ca4da (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:52:03.688 INFO:journalctl@ceph.osd.0.smithi064.stdout:Aug 17 17:52:03 smithi064 podman[348175]: 2024-08-17 17:52:03.437893751 +0000 UTC m=+0.581356213 container died cd3fe0cf64f18a013c21a2e7829418f5711fcd2b0a91f1abb446f591301ca4da (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:04.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.0.service' 2024-08-17T17:52:04.180 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:04.180 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-17T17:52:04.180 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-17T17:52:04.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.1 2024-08-17T17:52:04.406 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:04 smithi064 systemd[1]: Stopping Ceph osd.1 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:04.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:04 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:52:04.535+0000 7f10ff48c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:04.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:04 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:52:04.535+0000 7f10ff48c640 -1 osd.1 544 *** Got signal Terminated *** 2024-08-17T17:52:04.688 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:04 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1[125627]: 2024-08-17T17:52:04.535+0000 7f10ff48c640 -1 osd.1 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:08.161 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.840Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.842Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.842Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.843Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.845Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:08.162 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:52:07 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:52:07.845Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2024-08-17T17:52:10.337 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:10 smithi064 podman[348280]: 2024-08-17 17:52:10.045037209 +0000 UTC m=+5.638013055 container died b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:52:10.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:10 smithi064 podman[348280]: 2024-08-17 17:52:10.519066666 +0000 UTC m=+6.112042553 container cleanup b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:10.687 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:10 smithi064 bash[348280]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:52:11.158 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:10 smithi064 podman[348310]: 2024-08-17 17:52:10.804101681 +0000 UTC m=+0.752541639 container remove b136792f5b7c0d52df4f4d09d8b7af91a277d24ee429619676d54262ec09fc66 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-17T17:52:11.437 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.061230286 +0000 UTC m=+0.017697889 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:52:11.437 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.180306192 +0000 UTC m=+0.136773812 container create 4afbc3f5c97f9c73e023889ab139546a6a81fc30dd88b501d653d774a7e4b021 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:11.936 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.44107171 +0000 UTC m=+0.397539302 container init 4afbc3f5c97f9c73e023889ab139546a6a81fc30dd88b501d653d774a7e4b021 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T17:52:11.936 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.445661591 +0000 UTC m=+0.402129172 container start 4afbc3f5c97f9c73e023889ab139546a6a81fc30dd88b501d653d774a7e4b021 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:11.936 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.541056779 +0000 UTC m=+0.497524380 container attach 4afbc3f5c97f9c73e023889ab139546a6a81fc30dd88b501d653d774a7e4b021 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe) 2024-08-17T17:52:11.936 INFO:journalctl@ceph.osd.1.smithi064.stdout:Aug 17 17:52:11 smithi064 podman[348397]: 2024-08-17 17:52:11.638336775 +0000 UTC m=+0.594804353 container died 4afbc3f5c97f9c73e023889ab139546a6a81fc30dd88b501d653d774a7e4b021 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T17:52:12.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.1.service' 2024-08-17T17:52:12.368 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:12.368 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-17T17:52:12.369 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-17T17:52:12.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.2 2024-08-17T17:52:12.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:12 smithi064 systemd[1]: Stopping Ceph osd.2 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:13.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:52:12.710+0000 7f2decefd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:13.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:52:12.710+0000 7f2decefd640 -1 osd.2 544 *** Got signal Terminated *** 2024-08-17T17:52:13.188 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:12 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2[130627]: 2024-08-17T17:52:12.710+0000 7f2decefd640 -1 osd.2 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:18.687 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:18 smithi064 podman[348499]: 2024-08-17 17:52:18.410738847 +0000 UTC m=+5.808472682 container died 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:52:19.174 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:18 smithi064 podman[348499]: 2024-08-17 17:52:18.906127863 +0000 UTC m=+6.303861699 container cleanup 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True) 2024-08-17T17:52:19.174 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:18 smithi064 bash[348499]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:52:19.437 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348517]: 2024-08-17 17:52:19.1742917 +0000 UTC m=+0.765371974 container remove 8a5e7a009d592eccd5c4349cf1cbf9340d7ebee4609d741ba4acdfbdeac0be9c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:52:19.836 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.401132415 +0000 UTC m=+0.017970459 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:52:19.836 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.516953363 +0000 UTC m=+0.133791393 container create 5f4bbb17834751e4b1408603d566ac368d33c3175def243bb873f4cab707fa20 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:52:19.836 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.735772869 +0000 UTC m=+0.352610902 container init 5f4bbb17834751e4b1408603d566ac368d33c3175def243bb873f4cab707fa20 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T17:52:19.837 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.739348693 +0000 UTC m=+0.356186733 container start 5f4bbb17834751e4b1408603d566ac368d33c3175def243bb873f4cab707fa20 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:52:19.837 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.835856727 +0000 UTC m=+0.452694772 container attach 5f4bbb17834751e4b1408603d566ac368d33c3175def243bb873f4cab707fa20 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:20.187 INFO:journalctl@ceph.osd.2.smithi064.stdout:Aug 17 17:52:19 smithi064 podman[348604]: 2024-08-17 17:52:19.931833136 +0000 UTC m=+0.548671177 container died 5f4bbb17834751e4b1408603d566ac368d33c3175def243bb873f4cab707fa20 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:52:20.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.2.service' 2024-08-17T17:52:20.541 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:20.542 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-17T17:52:20.542 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-17T17:52:20.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.3 2024-08-17T17:52:20.918 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:20 smithi064 systemd[1]: Stopping Ceph osd.3 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:21.187 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:20.917+0000 7f97ace93640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:21.187 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:20.917+0000 7f97ace93640 -1 osd.3 544 *** Got signal Terminated *** 2024-08-17T17:52:21.188 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:20 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:20.917+0000 7f97ace93640 -1 osd.3 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:22.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:22 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:22.410+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:22.687 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:22 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:22.401+0000 7f97a94a8640 -1 osd.3 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:00.320255+0000 front 2024-08-17T17:52:00.320336+0000 (oldest deadline 2024-08-17T17:52:22.020420+0000) 2024-08-17T17:52:23.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:23.101+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:23.687 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:23 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:23.386+0000 7f97a94a8640 -1 osd.3 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:00.320255+0000 front 2024-08-17T17:52:00.320336+0000 (oldest deadline 2024-08-17T17:52:22.020420+0000) 2024-08-17T17:52:23.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:23 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:23.443+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:24.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:24.124+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:24.687 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:24 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:24.338+0000 7f97a94a8640 -1 osd.3 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:00.320255+0000 front 2024-08-17T17:52:00.320336+0000 (oldest deadline 2024-08-17T17:52:22.020420+0000) 2024-08-17T17:52:24.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:24 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:24.464+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:25.501 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:25 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:25.161+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:25.687 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:25 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3[135635]: 2024-08-17T17:52:25.364+0000 7f97a94a8640 -1 osd.3 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:00.320255+0000 front 2024-08-17T17:52:00.320336+0000 (oldest deadline 2024-08-17T17:52:22.020420+0000) 2024-08-17T17:52:25.818 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:25 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:25.500+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:26.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:25 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:25.817+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:26.473 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:26.160+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:26.660 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:26 smithi064 podman[348706]: 2024-08-17 17:52:26.3670817 +0000 UTC m=+5.582053449 container died e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-17T17:52:26.830 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:26.472+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:26.937 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:26 smithi064 podman[348706]: 2024-08-17 17:52:26.733764954 +0000 UTC m=+5.948736707 container cleanup e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, GIT_CLEAN=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T17:52:26.937 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:26 smithi064 bash[348706]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:52:27.120 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:26 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:26.830+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:27.310 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:26 smithi064 podman[348721]: 2024-08-17 17:52:26.976259195 +0000 UTC m=+0.611611820 container remove e09fd3b84f1ef13aedb4da2ac5700bbab79a032bbbe64d336ad380a47f222e48 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0) 2024-08-17T17:52:27.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:27.120+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:27.580 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.31027133 +0000 UTC m=+0.114086360 container create ba204c44c92c021ad7fdd10ebd8f3fe177d22c4cde0aaa5b06293040860e9839 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe) 2024-08-17T17:52:27.580 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.21370761 +0000 UTC m=+0.017522657 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:52:27.581 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.579915311 +0000 UTC m=+0.383730353 container init ba204c44c92c021ad7fdd10ebd8f3fe177d22c4cde0aaa5b06293040860e9839 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, ceph=True) 2024-08-17T17:52:27.847 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:27.442+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:27.937 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.583717667 +0000 UTC m=+0.387532700 container start ba204c44c92c021ad7fdd10ebd8f3fe177d22c4cde0aaa5b06293040860e9839 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:52:27.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.654533115 +0000 UTC m=+0.458348158 container attach ba204c44c92c021ad7fdd10ebd8f3fe177d22c4cde0aaa5b06293040860e9839 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:52:27.938 INFO:journalctl@ceph.osd.3.smithi064.stdout:Aug 17 17:52:27 smithi064 podman[348814]: 2024-08-17 17:52:27.783609916 +0000 UTC m=+0.587424943 container died ba204c44c92c021ad7fdd10ebd8f3fe177d22c4cde0aaa5b06293040860e9839 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:52:28.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:27.847+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:28.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:28.084+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:28.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:27 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:27.958+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:28.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.3.service' 2024-08-17T17:52:28.478 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:28.478 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-17T17:52:28.478 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-17T17:52:28.479 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.4 2024-08-17T17:52:28.806 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:28 smithi175 systemd[1]: Stopping Ceph osd.4 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:28.807 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:28.805+0000 7fc4ef7f3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:28.807 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:28.474+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:29.070 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:28.941+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:29.071 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:28.805+0000 7fc4ef7f3640 -1 osd.4 544 *** Got signal Terminated *** 2024-08-17T17:52:29.071 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:28.805+0000 7fc4ef7f3640 -1 osd.4 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:29.071 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:28 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:28.883+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:29.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:29.069+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:29.851 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:29.472+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:30.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:30 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:30.089+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:30.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:29.850+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:30.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:29 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:29.899+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:30.835 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:30 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:30.485+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:30.835 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:30 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:30.485+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:31.136 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:30 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:30.834+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:31.137 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:30 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:30.894+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:31.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:31.135+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:31.806 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:31.463+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:31.806 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:31.463+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:32.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:31.804+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:32.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:31.919+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:32.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:31 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:31.919+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:32.454 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:32.171+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:32.767 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:32.453+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:32.767 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:32.453+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:33.160 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:32.766+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:33.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:32.965+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:33.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:32 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:32.965+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:33.660 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:33.434+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:33.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:33.434+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:33.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:33.198+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:33.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:33.198+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:34.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4[39434]: 2024-08-17T17:52:33.778+0000 7fc4ebe07640 -1 osd.4 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.005199+0000 front 2024-08-17T17:52:01.005116+0000 (oldest deadline 2024-08-17T17:52:25.005337+0000) 2024-08-17T17:52:34.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:33.922+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:34.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:33 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:33.922+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:34.456 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:34.208+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:34.456 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:34.208+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:34.746 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:34 smithi175 podman[252691]: 2024-08-17 17:52:34.483668534 +0000 UTC m=+5.804680968 container died 8eb7c8820c012c1080e1632dd72853ae6d750aec84c5498aa2013a344d2fc4fc (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:52:34.747 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:34.455+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:34.747 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:34.455+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:35.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:34 smithi175 podman[252691]: 2024-08-17 17:52:34.922268837 +0000 UTC m=+6.243281271 container cleanup 8eb7c8820c012c1080e1632dd72853ae6d750aec84c5498aa2013a344d2fc4fc (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:35.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:34 smithi175 bash[252691]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4 2024-08-17T17:52:35.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:34.947+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:35.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:34 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:34.947+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:35.497 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252706]: 2024-08-17 17:52:35.173798834 +0000 UTC m=+0.685873582 container remove 8eb7c8820c012c1080e1632dd72853ae6d750aec84c5498aa2013a344d2fc4fc (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-17T17:52:35.498 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:35.172+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:35.498 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:35.172+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:35.828 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:35.496+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:35.828 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:35.496+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:35.828 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252793]: 2024-08-17 17:52:35.498731041 +0000 UTC m=+0.017409462 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:52:35.828 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252793]: 2024-08-17 17:52:35.599927874 +0000 UTC m=+0.118606332 container create b427e33b03dd592a8c9075c5aae2d5090971aa129d6103f8198483ca3c3f10d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4-deactivate, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:52:36.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:35.917+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:36.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:35 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:35.917+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:36.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252793]: 2024-08-17 17:52:35.827762018 +0000 UTC m=+0.346440430 container init b427e33b03dd592a8c9075c5aae2d5090971aa129d6103f8198483ca3c3f10d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:52:36.161 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252793]: 2024-08-17 17:52:35.831403927 +0000 UTC m=+0.350082349 container start b427e33b03dd592a8c9075c5aae2d5090971aa129d6103f8198483ca3c3f10d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-17T17:52:36.162 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:35 smithi175 podman[252793]: 2024-08-17 17:52:35.935590397 +0000 UTC m=+0.454268813 container attach b427e33b03dd592a8c9075c5aae2d5090971aa129d6103f8198483ca3c3f10d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:36.162 INFO:journalctl@ceph.osd.4.smithi175.stdout:Aug 17 17:52:36 smithi175 podman[252793]: 2024-08-17 17:52:36.023135849 +0000 UTC m=+0.541814262 container died b427e33b03dd592a8c9075c5aae2d5090971aa129d6103f8198483ca3c3f10d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:52:36.476 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:36.185+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:36.476 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:36.185+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:36.616 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.4.service' 2024-08-17T17:52:36.654 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:36.655 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-17T17:52:36.655 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T17:52:36.655 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:52:36.740 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:36.541+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:36.740 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:36.541+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:37.069 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:36 smithi175 systemd[1]: Stopping Ceph osd.5 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:37.069 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:36.911+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:37.069 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:36 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:36.911+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:37.069 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:37.067+0000 7f925aee1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:37.069 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:37.067+0000 7f925aee1640 -1 osd.5 544 *** Got signal Terminated *** 2024-08-17T17:52:37.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:37.067+0000 7f925aee1640 -1 osd.5 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:37.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:37.158+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:37.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:37.158+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:37.895 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:37.505+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:37.895 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:37.505+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:38.160 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:37.894+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:38.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:37 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:37.894+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:38.475 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:38.168+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:38.475 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:38.168+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:38.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:38.474+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:38.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:38.474+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:39.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:39.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:39.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:39.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:39.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:38.932+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:39.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:38.932+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:39.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:38 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:38.932+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:14.506293+0000 front 2024-08-17T17:52:14.506277+0000 (oldest deadline 2024-08-17T17:52:38.606151+0000) 2024-08-17T17:52:39.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:39.458+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:39.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:39.458+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:40.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:40.115+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:40.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:40.115+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:40.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:40.115+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:40.411 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:39.961+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:40.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:39.961+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:40.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:39 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:39.961+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:14.506293+0000 front 2024-08-17T17:52:14.506277+0000 (oldest deadline 2024-08-17T17:52:38.606151+0000) 2024-08-17T17:52:40.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:40.483+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:40.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:40.483+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:41.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:41.071+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:41.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:41.071+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:41.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:41.071+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:41.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:40.993+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:41.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:40.993+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:41.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:40 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:40.993+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:14.506293+0000 front 2024-08-17T17:52:14.506277+0000 (oldest deadline 2024-08-17T17:52:38.606151+0000) 2024-08-17T17:52:41.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:41.436+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:41.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:41.436+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:41.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:41 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:41.436+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:42.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:42.072+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:42.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:42.072+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:42.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:42.072+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:42.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:42.002+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:52:01.104418+0000 front 2024-08-17T17:52:01.104505+0000 (oldest deadline 2024-08-17T17:52:27.004344+0000) 2024-08-17T17:52:42.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:42.002+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.004988+0000 front 2024-08-17T17:52:07.004961+0000 (oldest deadline 2024-08-17T17:52:31.005247+0000) 2024-08-17T17:52:42.412 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5[158968]: 2024-08-17T17:52:42.002+0000 7f92574f5640 -1 osd.5 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:14.506293+0000 front 2024-08-17T17:52:14.506277+0000 (oldest deadline 2024-08-17T17:52:38.606151+0000) 2024-08-17T17:52:42.868 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 podman[252907]: 2024-08-17 17:52:42.545132591 +0000 UTC m=+5.639065380 container died bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:42.869 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:42.482+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:42.869 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:42.482+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:42.869 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:42 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:42.482+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:43.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 podman[252907]: 2024-08-17 17:52:42.967562262 +0000 UTC m=+6.061495062 container cleanup bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:43.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:42 smithi175 bash[252907]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5 2024-08-17T17:52:43.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.026+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:43.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.026+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:43.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.026+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:43.578 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[252925]: 2024-08-17 17:52:43.252600904 +0000 UTC m=+0.657522786 container remove bf23e664d71634fd35279485d70ce662e43893c169a88183d7e85ec5fe745bac (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:52:43.579 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:43.442+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:43.579 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:43.442+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:43.579 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:43.442+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:43.898 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[253015]: 2024-08-17 17:52:43.578481395 +0000 UTC m=+0.106685059 container create 7f3d4f14461f65c286dacd8f99d26af1bb956b1f6cc9e667cf25e01be5027757 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:52:43.898 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[253015]: 2024-08-17 17:52:43.489909009 +0000 UTC m=+0.018112635 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:52:43.899 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[253015]: 2024-08-17 17:52:43.898078012 +0000 UTC m=+0.426281647 container init 7f3d4f14461f65c286dacd8f99d26af1bb956b1f6cc9e667cf25e01be5027757 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:52:44.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[253015]: 2024-08-17 17:52:43.901427779 +0000 UTC m=+0.429631397 container start 7f3d4f14461f65c286dacd8f99d26af1bb956b1f6cc9e667cf25e01be5027757 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:52:44.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:43 smithi175 podman[253015]: 2024-08-17 17:52:43.981199146 +0000 UTC m=+0.509402767 container attach 7f3d4f14461f65c286dacd8f99d26af1bb956b1f6cc9e667cf25e01be5027757 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:44.161 INFO:journalctl@ceph.osd.5.smithi175.stdout:Aug 17 17:52:44 smithi175 podman[253015]: 2024-08-17 17:52:44.089344586 +0000 UTC m=+0.617548205 container died 7f3d4f14461f65c286dacd8f99d26af1bb956b1f6cc9e667cf25e01be5027757 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f) 2024-08-17T17:52:44.162 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.987+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:44.162 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.987+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:44.162 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:43 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:43.987+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:44.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:44 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:44.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:44.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:44 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:44.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:44.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:44 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:44.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:44.729 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T17:52:44.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:44.769 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T17:52:44.770 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:52:44.770 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:52:45.007 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:44 smithi175 systemd[1]: Stopping Ceph osd.6 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:45.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.095+0000 7f1129e3b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:45.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.095+0000 7f1129e3b640 -1 osd.6 544 *** Got signal Terminated *** 2024-08-17T17:52:45.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.095+0000 7f1129e3b640 -1 osd.6 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:45.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:45.006+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:45.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:45.006+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:45.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:45.006+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:45.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:45.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:45.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:45 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:45.419+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:46.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:46.039+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:46.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:46.039+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:46.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:46.039+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:46.749 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:46.423+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:46.749 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:46.423+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:46.749 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:46 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:46.423+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:47.350 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:47.076+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:47.350 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:47.076+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:47.350 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:47.076+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:47.350 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:47.076+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:47.660 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:47.428+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:47.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:47.428+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:47.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:47.428+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:47.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:47 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:47.428+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.480825+0000 front 2024-08-17T17:52:25.480872+0000 (oldest deadline 2024-08-17T17:52:47.180777+0000) 2024-08-17T17:52:48.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:48.043+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:48.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:48.043+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:48.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:48.043+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:48.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:48.043+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:48.910 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:48.420+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:48.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:48.420+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:48.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:48.420+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:48.911 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:48 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:48.420+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.480825+0000 front 2024-08-17T17:52:25.480872+0000 (oldest deadline 2024-08-17T17:52:47.180777+0000) 2024-08-17T17:52:49.387 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:49.083+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:49.387 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:49.083+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:49.387 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:49.083+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:49.387 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:49.083+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:49.660 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:49.386+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.377675+0000 front 2024-08-17T17:51:57.377776+0000 (oldest deadline 2024-08-17T17:52:22.077707+0000) 2024-08-17T17:52:49.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:49.386+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:07.378665+0000 front 2024-08-17T17:52:07.378636+0000 (oldest deadline 2024-08-17T17:52:30.278581+0000) 2024-08-17T17:52:49.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:49.386+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.079933+0000 front 2024-08-17T17:52:16.080030+0000 (oldest deadline 2024-08-17T17:52:41.379905+0000) 2024-08-17T17:52:49.661 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:49 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6[160813]: 2024-08-17T17:52:49.386+0000 7f112644f640 -1 osd.6 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.480825+0000 front 2024-08-17T17:52:25.480872+0000 (oldest deadline 2024-08-17T17:52:47.180777+0000) 2024-08-17T17:52:50.410 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:50 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:50.041+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:50.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:50 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:50.041+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:50.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:50 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:50.041+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:50.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:50 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:50.041+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:51.006 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:50 smithi175 podman[253119]: 2024-08-17 17:52:50.737735802 +0000 UTC m=+5.747028050 container died 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:51.380 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253119]: 2024-08-17 17:52:51.103847477 +0000 UTC m=+6.113139731 container cleanup 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0) 2024-08-17T17:52:51.380 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 bash[253119]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:52:51.381 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:51.057+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:51.381 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:51.057+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:51.381 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:51.057+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:51.381 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:51 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:51.057+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:51.660 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253136]: 2024-08-17 17:52:51.380075779 +0000 UTC m=+0.640321543 container remove 99382b21fe58d0c26c836ffd2c2a66647a7b5ff7105d08ddc6ac0f63672e857e (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6, ceph=True, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:52:51.967 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253223]: 2024-08-17 17:52:51.601542422 +0000 UTC m=+0.017686149 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:52:51.967 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253223]: 2024-08-17 17:52:51.714302902 +0000 UTC m=+0.130446617 container create 1bc983eb0b30c09fea1e1257dd9c0d0900d3770c150bf2b7ef33d2eeed2b8b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:52:51.967 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253223]: 2024-08-17 17:52:51.966967136 +0000 UTC m=+0.383110855 container init 1bc983eb0b30c09fea1e1257dd9c0d0900d3770c150bf2b7ef33d2eeed2b8b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:52:52.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:51 smithi175 podman[253223]: 2024-08-17 17:52:51.970244777 +0000 UTC m=+0.386388497 container start 1bc983eb0b30c09fea1e1257dd9c0d0900d3770c150bf2b7ef33d2eeed2b8b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=reef-ced649f) 2024-08-17T17:52:52.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:52 smithi175 podman[253223]: 2024-08-17 17:52:52.050063644 +0000 UTC m=+0.466207360 container attach 1bc983eb0b30c09fea1e1257dd9c0d0900d3770c150bf2b7ef33d2eeed2b8b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-08-17T17:52:52.411 INFO:journalctl@ceph.osd.6.smithi175.stdout:Aug 17 17:52:52 smithi175 podman[253223]: 2024-08-17 17:52:52.158030449 +0000 UTC m=+0.574174177 container died 1bc983eb0b30c09fea1e1257dd9c0d0900d3770c150bf2b7ef33d2eeed2b8b87 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T17:52:52.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:52.070+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:52.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:52.070+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:52.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:52.070+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:52.412 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:52 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:52.070+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:52.735 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:52:52.774 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:52:52.774 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:52:52.774 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:52:52.775 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:52:53.098 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:52 smithi175 systemd[1]: Stopping Ceph osd.7 for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:52:53.410 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.096+0000 7eff8b27a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.096+0000 7eff8b27a640 -1 osd.7 544 *** Got signal Terminated *** 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.096+0000 7eff8b27a640 -1 osd.7 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.098+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.098+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.098+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:53.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:53 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:53.098+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:54.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:54.128+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:54.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:54.128+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:54.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:54.128+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:54.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:54 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:54.128+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:55.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:55 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:55.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:55.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:55 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:55.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:55.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:55 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:55.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:55.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:55 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:55.152+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:56.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:56 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:56.182+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:56.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:56 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:56.182+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:56.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:56 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:56.182+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:56.661 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:56 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:56.182+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:57.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:57 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:57.151+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2024-08-17T17:51:57.952607+0000 front 2024-08-17T17:51:57.952573+0000 (oldest deadline 2024-08-17T17:52:22.652458+0000) 2024-08-17T17:52:57.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:57 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:57.151+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2024-08-17T17:52:08.454273+0000 front 2024-08-17T17:52:08.454187+0000 (oldest deadline 2024-08-17T17:52:32.554115+0000) 2024-08-17T17:52:57.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:57 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:57.151+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2024-08-17T17:52:16.655077+0000 front 2024-08-17T17:52:16.655085+0000 (oldest deadline 2024-08-17T17:52:39.554996+0000) 2024-08-17T17:52:57.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:57 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:57.151+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2024-08-17T17:52:25.257328+0000 front 2024-08-17T17:52:25.257276+0000 (oldest deadline 2024-08-17T17:52:46.957190+0000) 2024-08-17T17:52:57.411 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:57 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7[248431]: 2024-08-17T17:52:57.151+0000 7eff8788e640 -1 osd.7 544 heartbeat_check: no reply from 172.21.15.175:6804 osd.4 since back 2024-08-17T17:52:32.858165+0000 front 2024-08-17T17:52:32.858120+0000 (oldest deadline 2024-08-17T17:52:56.358033+0000) 2024-08-17T17:52:58.911 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:58 smithi175 podman[253324]: 2024-08-17 17:52:58.655847796 +0000 UTC m=+5.659809733 container died 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-17T17:52:59.281 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253324]: 2024-08-17 17:52:59.013753636 +0000 UTC m=+6.017715567 container cleanup 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-08-17T17:52:59.282 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 bash[253324]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:52:59.646 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253340]: 2024-08-17 17:52:59.281519443 +0000 UTC m=+0.623456150 container remove 4e641ff9b01ab993a1bb0854b1e3bfbabdc016d5ae6a56892120ec212f3bb7cb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:52:59.897 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253427]: 2024-08-17 17:52:59.549110368 +0000 UTC m=+0.017644626 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:52:59.897 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253427]: 2024-08-17 17:52:59.657217037 +0000 UTC m=+0.125751286 container create 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-17T17:52:59.897 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253427]: 2024-08-17 17:52:59.89340446 +0000 UTC m=+0.361938728 container init 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-08-17T17:52:59.897 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253427]: 2024-08-17 17:52:59.896711897 +0000 UTC m=+0.365246155 container start 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-08-17T17:53:00.160 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:52:59 smithi175 podman[253427]: 2024-08-17 17:52:59.976255604 +0000 UTC m=+0.444789864 container attach 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:53:00.161 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:53:00 smithi175 podman[253427]: 2024-08-17 17:53:00.078978153 +0000 UTC m=+0.547512403 container died 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:53:00.683 INFO:journalctl@ceph.osd.7.smithi175.stdout:Aug 17 17:53:00 smithi175 podman[253451]: 2024-08-17 17:53:00.665246276 +0000 UTC m=+0.580460251 container remove 8840c5811f4cfc3a30ebaf21d4d3fe3d08afe702d5fcf92c1d199daee8da6994 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-08-17T17:53:00.688 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:53:00.726 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:53:00.726 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:53:00.726 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-17T17:53:00.727 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a 2024-08-17T17:53:01.060 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:00 smithi175 systemd[1]: Stopping Ceph prometheus.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:53:01.340 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:53:01.341 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.059Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.061Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.061Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a[87380]: ts=2024-08-17T17:53:01.061Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T17:53:01.342 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 podman[253529]: 2024-08-17 17:53:01.069943949 +0000 UTC m=+0.113501276 container died 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:01.660 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 podman[253529]: 2024-08-17 17:53:01.417876041 +0000 UTC m=+0.461433399 container cleanup 02b6969fb42d8fe0168b4811987248cf4048a0bf36fba4cfb01b487552283d03 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:01.661 INFO:journalctl@ceph.prometheus.a.smithi175.stdout:Aug 17 17:53:01 smithi175 bash[253529]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:53:01.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@prometheus.a.service' 2024-08-17T17:53:01.881 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:53:01.881 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-17T17:53:01.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:53:02.045 INFO:teuthology.orchestra.run.smithi064.stdout:Deleting cluster with fsid: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:53:03.688 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:03 smithi064 systemd[1]: Stopping Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:53:04.116 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:03 smithi064 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a[99768]: ts=2024-08-17T17:53:03.767Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:53:04.116 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:03 smithi064 podman[349217]: 2024-08-17 17:53:03.818395763 +0000 UTC m=+0.176139934 container died f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:04.437 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 podman[349217]: 2024-08-17 17:53:04.283801267 +0000 UTC m=+0.641545480 container cleanup f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:04.437 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 bash[349217]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:53:04.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 podman[349231]: 2024-08-17 17:53:04.551784347 +0000 UTC m=+0.727935109 container remove f2848ae5f362c291eee1249e29b41d87951fe3b1b31b24ab978cb0abc0813d2f (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:04.937 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 podman[349231]: 2024-08-17 17:53:04.628011876 +0000 UTC m=+0.804162639 volume remove 6861d97d78d0d1b117c4047df2afc23745c7d9fbf5050818c8624b5a64c9e4ce 2024-08-17T17:53:04.938 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:53:04.938 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 systemd[1]: Stopped Ceph alertmanager.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:53:04.938 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Aug 17 17:53:04 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 3.186s CPU time. 2024-08-17T17:53:05.225 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:04 smithi064 systemd[1]: Stopping Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:53:05.630 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 podman[349338]: 2024-08-17 17:53:05.224216787 +0000 UTC m=+0.117399129 container died 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:05.630 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 podman[349338]: 2024-08-17 17:53:05.630030583 +0000 UTC m=+0.523212967 container cleanup 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:05.937 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 bash[349338]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:53:05.937 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 podman[349351]: 2024-08-17 17:53:05.847849368 +0000 UTC m=+0.623096133 container remove 7612fc9a72e1a2949e76214e84d30e7403f654046b1a39b68e6de66e6c2cfab6 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:53:05.937 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:53:06.435 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:53:06.435 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 systemd[1]: Stopped Ceph node-exporter.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:53:06.435 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Aug 17 17:53:05 smithi064 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 7.652s CPU time. 2024-08-17T17:53:10.907 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:53:11.064 INFO:teuthology.orchestra.run.smithi175.stdout:Deleting cluster with fsid: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T17:53:12.802 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:12 smithi175 systemd[1]: Stopping Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:53:13.056 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:12 smithi175 podman[253952]: 2024-08-17 17:53:12.807430848 +0000 UTC m=+0.117816806 container died ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:53:13.406 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 podman[253952]: 2024-08-17 17:53:13.155246463 +0000 UTC m=+0.465632420 container cleanup ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:53:13.407 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 bash[253952]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:53:13.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 podman[253967]: 2024-08-17 17:53:13.406511006 +0000 UTC m=+0.602098284 container remove ee4f9a740541a5aadc18bd43cad3fb00f75d03e368ee5f16f7a5d9024ee90ef9 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:53:13.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:53:13.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:53:13.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 systemd[1]: Stopped Ceph node-exporter.b for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:53:13.661 INFO:journalctl@ceph.node-exporter.b.smithi175.stdout:Aug 17 17:53:13 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 6.643s CPU time. 2024-08-17T17:53:14.018 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:13 smithi175 systemd[1]: Stopping Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968... 2024-08-17T17:53:14.326 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=server t=2024-08-17T17:53:14.018201292Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-08-17T17:53:14.327 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=ticker t=2024-08-17T17:53:14.018548159Z level=info msg=stopped last_tick=2024-08-17T17:53:10Z 2024-08-17T17:53:14.327 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=tracing t=2024-08-17T17:53:14.018557184Z level=info msg="Closing tracing" 2024-08-17T17:53:14.327 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a[92297]: logger=grafana-apiserver t=2024-08-17T17:53:14.018601656Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-08-17T17:53:14.327 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 podman[254071]: 2024-08-17 17:53:14.038588478 +0000 UTC m=+0.126657283 container died 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:53:14.660 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 podman[254071]: 2024-08-17 17:53:14.41770932 +0000 UTC m=+0.505778115 container cleanup 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:53:14.661 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 bash[254071]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:53:15.018 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 podman[254085]: 2024-08-17 17:53:14.677512954 +0000 UTC m=+0.632068624 container remove 92fb2e1fda056741fd51eb3f74b9419659b26ad9db5d91722a1c0020b7028d74 (image=quay.io/ceph/grafana:10.4.0, name=ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:53:15.018 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-17T17:53:15.018 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 systemd[1]: Stopped Ceph grafana.a for d49b6e42-5cb9-11ef-bccf-c7b262605968. 2024-08-17T17:53:15.018 INFO:journalctl@ceph.grafana.a.smithi175.stdout:Aug 17 17:53:14 smithi175 systemd[1]: ceph-d49b6e42-5cb9-11ef-bccf-c7b262605968@grafana.a.service: Consumed 28.250s CPU time. 2024-08-17T17:53:19.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:53:19.593 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:53:19.620 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-17T17:53:19.622 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi064/crash 2024-08-17T17:53:19.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi064.stderr:tar: /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi064.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:53:19.679 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi175/crash 2024-08-17T17:53:19.680 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi175.stderr:tar: /var/lib/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi175.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:53:19.723 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-17T17:53:19.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-08-17T17:53:19.812 INFO:teuthology.orchestra.run.smithi064.stdout:2024-08-17T17:04:24.294004+0000 mon.a (mon.0) 693 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:53:19.826 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-08-17T17:53:19.826 INFO:tasks.cephadm:Compressing logs... 2024-08-17T17:53:19.826 DEBUG:teuthology.orchestra.run.smithi064:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:53:19.829 DEBUG:teuthology.orchestra.run.smithi175:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:53:19.860 INFO:teuthology.orchestra.run.smithi064.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory 2024-08-17T17:53:19.860 INFO:teuthology.orchestra.run.smithi064.stderr: -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:53:19.860 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.2.log 2024-08-17T17:53:19.863 INFO:teuthology.orchestra.run.smithi064.stderr:gzip/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log: -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:53:19.863 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.3.log 2024-08-17T17:53:19.863 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi064.qahkwa.log 2024-08-17T17:53:19.863 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.1.log 2024-08-17T17:53:19.863 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi064.qahkwa.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.y.log 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr: 85.8% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi064.qahkwa.log.gz 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.0.log/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.1.log: 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.c.log 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.a.log 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:53:19.864 INFO:teuthology.orchestra.run.smithi064.stderr:gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:53:19.866 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.a.log: /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.7.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.5.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.4.log 2024-08-17T17:53:19.869 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi175.nhcgie.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.x.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi175.nhcgie.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.6.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr: 83.5% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mds.foo.smithi175.nhcgie.log.gz 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.b.log 2024-08-17T17:53:19.870 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:53:19.871 INFO:teuthology.orchestra.run.smithi064.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 92.7% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:53:20.103 INFO:teuthology.orchestra.run.smithi064.stderr: 95.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:53:20.177 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 90.6% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:53:20.232 INFO:teuthology.orchestra.run.smithi175.stderr: 86.9% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:53:20.279 INFO:teuthology.orchestra.run.smithi064.stderr: 91.3% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:53:20.280 INFO:teuthology.orchestra.run.smithi064.stderr: 94.3% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:53:20.303 INFO:teuthology.orchestra.run.smithi175.stderr: 85.7% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:53:20.384 INFO:teuthology.orchestra.run.smithi175.stderr: 89.8% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.x.log.gz 2024-08-17T17:53:20.449 INFO:teuthology.orchestra.run.smithi175.stderr: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:53:20.626 INFO:teuthology.orchestra.run.smithi064.stderr: 91.9% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:53:20.980 INFO:teuthology.orchestra.run.smithi175.stderr: 91.2% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:53:28.003 INFO:teuthology.orchestra.run.smithi064.stderr: 91.9% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.c.log.gz 2024-08-17T17:53:30.889 INFO:teuthology.orchestra.run.smithi064.stderr: 89.6% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mgr.y.log.gz 2024-08-17T17:53:31.232 INFO:teuthology.orchestra.run.smithi175.stderr: 92.3% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.b.log.gz 2024-08-17T17:54:06.643 INFO:teuthology.orchestra.run.smithi064.stderr: 90.3% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-mon.a.log.gz 2024-08-17T18:03:06.852 INFO:teuthology.orchestra.run.smithi175.stderr: 91.8% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.4.log.gz 2024-08-17T18:03:57.365 INFO:teuthology.orchestra.run.smithi175.stderr: 93.8% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.6.log.gz 2024-08-17T18:06:03.914 INFO:teuthology.orchestra.run.smithi175.stderr: 93.4% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.7.log.gz 2024-08-17T18:06:08.831 INFO:teuthology.orchestra.run.smithi175.stderr: 93.7% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.5.log.gz 2024-08-17T18:06:08.834 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-08-17T18:06:08.834 INFO:teuthology.orchestra.run.smithi175.stderr:real 12m48.989s 2024-08-17T18:06:08.834 INFO:teuthology.orchestra.run.smithi175.stderr:user 5m45.750s 2024-08-17T18:06:08.834 INFO:teuthology.orchestra.run.smithi175.stderr:sys 0m16.854s 2024-08-17T18:10:03.330 INFO:teuthology.orchestra.run.smithi064.stderr: 93.0% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.1.log.gz 2024-08-17T18:12:17.311 INFO:teuthology.orchestra.run.smithi064.stderr: 93.3% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.2.log.gz 2024-08-17T18:14:22.963 INFO:teuthology.orchestra.run.smithi064.stderr: 93.0% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.3.log.gz 2024-08-17T18:15:09.955 INFO:teuthology.orchestra.run.smithi064.stderr: 92.6% -- replaced with /var/log/ceph/d49b6e42-5cb9-11ef-bccf-c7b262605968/ceph-osd.0.log.gz 2024-08-17T18:15:09.957 INFO:teuthology.orchestra.run.smithi064.stderr: 2024-08-17T18:15:09.957 INFO:teuthology.orchestra.run.smithi064.stderr:real 21m50.113s 2024-08-17T18:15:09.957 INFO:teuthology.orchestra.run.smithi064.stderr:user 9m13.164s 2024-08-17T18:15:09.957 INFO:teuthology.orchestra.run.smithi064.stderr:sys 0m26.349s 2024-08-17T18:15:09.958 INFO:tasks.cephadm:Archiving logs... 2024-08-17T18:15:09.958 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi064/log 2024-08-17T18:15:09.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:16:31.660 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi175/log 2024-08-17T18:16:31.677 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:17:27.103 INFO:tasks.cephadm:Removing cluster... 2024-08-17T18:17:27.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 --force 2024-08-17T18:17:27.304 INFO:teuthology.orchestra.run.smithi064.stdout:Deleting cluster with fsid: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T18:17:28.145 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d49b6e42-5cb9-11ef-bccf-c7b262605968 --force 2024-08-17T18:17:28.504 INFO:teuthology.orchestra.run.smithi175.stdout:Deleting cluster with fsid: d49b6e42-5cb9-11ef-bccf-c7b262605968 2024-08-17T18:17:29.187 INFO:tasks.cephadm:Removing cephadm ... 2024-08-17T18:17:29.187 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:17:29.207 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:17:29.222 INFO:tasks.cephadm:Teardown complete 2024-08-17T18:17:29.226 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-08-17T18:17:29.237 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-08-17T18:17:29.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:17:29.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:17:29.359 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-17T18:17:29.369 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-17T18:17:29.369 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:17:29.372 DEBUG:teuthology.orchestra.run.smithi175:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:17:29.386 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:17:29.387 INFO:teuthology.orchestra.run.smithi064.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:17:29.411 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:17:29.412 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2024-08-17T18:17:29.412 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +387us[ +386us] +/- 53ms 2024-08-17T18:17:29.412 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 53 -324us[ -326us] +/- 62ms 2024-08-17T18:17:29.412 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 -249us[ -251us] +/- 75ms 2024-08-17T18:17:29.412 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:17:29.415 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:17:29.415 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-08-17T18:17:29.415 INFO:teuthology.orchestra.run.smithi175.stdout:^* hv01.front.sepia.ceph.com 3 6 377 54 +423us[ +432us] +/- 53ms 2024-08-17T18:17:29.415 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 -288us[ -288us] +/- 62ms 2024-08-17T18:17:29.416 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 -229us[ -229us] +/- 75ms 2024-08-17T18:17:29.416 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:17:29.417 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-17T18:17:29.462 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-17T18:17:29.462 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-17T18:17:29.499 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:17:29.641 DEBUG:teuthology.orchestra.run.smithi175:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:17:29.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-17T18:17:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.782 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-17T18:17:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.810 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-17T18:17:29.821 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-17T18:17:29.852 INFO:teuthology.task.internal:Duration was 5192.659805 seconds 2024-08-17T18:17:29.853 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-17T18:17:29.898 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-17T18:17:29.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:17:29.902 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:17:29.953 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:17:29.972 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:17:30.236 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-17T18:17:30.236 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2024-08-17T18:17:30.237 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:17:30.263 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi175.front.sepia.ceph.com 2024-08-17T18:17:30.264 DEBUG:teuthology.orchestra.run.smithi175:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:17:30.314 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-17T18:17:30.315 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:17:30.317 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:17:30.345 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-17T18:17:30.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:17:30.388 DEBUG:teuthology.orchestra.run.smithi175:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:17:32.418 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-17T18:17:32.430 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-17T18:17:32.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:17:32.458 DEBUG:teuthology.orchestra.run.smithi175:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:17:32.495 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-17T18:17:32.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:17:32.509 DEBUG:teuthology.orchestra.run.smithi175:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:17:32.545 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2024-08-17T18:17:32.569 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = core 2024-08-17T18:17:32.589 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:17:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.628 DEBUG:teuthology.orchestra.run.smithi175:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:17:32.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.646 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-17T18:17:32.656 INFO:teuthology.task.internal:Transferring archived files... 2024-08-17T18:17:32.657 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi064 2024-08-17T18:17:32.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:17:32.755 DEBUG:teuthology.misc:Transferring archived files from smithi175:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858331/remote/smithi175 2024-08-17T18:17:32.755 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:17:32.802 INFO:teuthology.task.internal:Removing archive directory... 2024-08-17T18:17:32.803 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:17:32.805 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:17:32.850 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-17T18:17:32.862 INFO:teuthology.task.internal:Not uploading archives. 2024-08-17T18:17:32.862 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-17T18:17:32.896 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-17T18:17:32.896 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:17:32.899 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:17:32.915 INFO:teuthology.orchestra.run.smithi175.stdout: 265306 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 18:17 /home/ubuntu/cephtest 2024-08-17T18:17:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: 266041 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:17 /home/ubuntu/cephtest 2024-08-17T18:17:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: 4587528 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:49 /home/ubuntu/cephtest/mnt.0 2024-08-17T18:17:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: 4587532 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:49 /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T18:17:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: 4587526 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 17:49 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T18:17:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.920 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-08-17T18:17:32.920 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:17:32.927 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-17T18:17:32.962 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-17T18:17:33.057 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:17:33.061 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5192.659805297852 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=500ba2cef4d640c4ae6d6d52ae9b3c57 status: fail success: false 2024-08-17T18:17:33.061 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T18:17:33.349 INFO:teuthology.run:FAIL